From e30246c49b353b9136f69caef23e7ba0e9df0f0e Mon Sep 17 00:00:00 2001 From: Chris Leary Date: Sun, 15 Oct 2017 20:17:11 -0700 Subject: [PATCH 001/573] [XLA] Make pad shape inference error more informative. PiperOrigin-RevId: 172276292 --- tensorflow/compiler/xla/service/shape_inference.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index f3c8e3aff3..6be6b77e85 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -464,7 +464,10 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, } if (ShapeUtil::Rank(operand_shape) != padding_config.dimensions_size()) { return InvalidArgument( - "the rank of the operand and the padding configuration do not match."); + "The rank of the operand and the padding configuration do not match: " + "%s vs %s", + ShapeUtil::HumanString(operand_shape).c_str(), + padding_config.ShortDebugString().c_str()); } if (operand_shape.element_type() != padding_value_shape.element_type()) { return InvalidArgument( -- GitLab From ca8af1d0dbb605087a4f8ae076188f2b9a26b1ba Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 15 Oct 2017 21:36:48 -0700 Subject: [PATCH 002/573] Replace NcclReduce/Broadcast ops during graph optimization so that we can generate gradients for Reduce/Broadcast. Changing _NcclBroadcastRecv shape input to int32 so that the corresponding Const op is outputting to HostMem. PiperOrigin-RevId: 172279684 --- tensorflow/contrib/nccl/BUILD | 2 + tensorflow/contrib/nccl/kernels/nccl_ops.cc | 28 +- .../contrib/nccl/kernels/nccl_rewrite.cc | 271 ++++++++++++++++++ tensorflow/contrib/nccl/ops/nccl_ops.cc | 84 ++++-- .../contrib/nccl/python/ops/nccl_ops.py | 138 ++++----- .../contrib/nccl/python/ops/nccl_ops_test.py | 87 +++--- 6 files changed, 483 insertions(+), 127 deletions(-) create mode 100644 tensorflow/contrib/nccl/kernels/nccl_rewrite.cc diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD index d6508362b8..5e7263ff62 100644 --- a/tensorflow/contrib/nccl/BUILD +++ b/tensorflow/contrib/nccl/BUILD @@ -71,10 +71,12 @@ tf_kernel_library( "kernels/nccl_manager.cc", "kernels/nccl_manager.h", "kernels/nccl_ops.cc", + "kernels/nccl_rewrite.cc", ], deps = [ "//tensorflow/core:framework", "//tensorflow/core:gpu_headers_lib", + "//tensorflow/core:proto_text", "@nccl_archive//:nccl", ], alwayslink = 1, diff --git a/tensorflow/contrib/nccl/kernels/nccl_ops.cc b/tensorflow/contrib/nccl/kernels/nccl_ops.cc index 4eb52492db..266d4f6f0d 100644 --- a/tensorflow/contrib/nccl/kernels/nccl_ops.cc +++ b/tensorflow/contrib/nccl/kernels/nccl_ops.cc @@ -15,8 +15,6 @@ limitations under the License. #if GOOGLE_CUDA -#include -#include #include #include "src/nccl.h" @@ -24,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" namespace tensorflow { +namespace { // Base class for all communicator ops that use nccl. // @@ -134,7 +133,7 @@ class NcclReduceSendKernel : public NcclReduceOpBase { compute_stream, &c->input(0), std::move(actual_done)); } }; -REGISTER_KERNEL_BUILDER(Name("NcclReduceSend").Device(DEVICE_GPU), +REGISTER_KERNEL_BUILDER(Name("_NcclReduceSend").Device(DEVICE_GPU), NcclReduceSendKernel); // To execute a single reduce, this kernel is called once for one devices, and @@ -166,7 +165,7 @@ class NcclReduceRecvKernel : public NcclReduceOpBase { private: ncclRedOp_t reduction_op_; }; -REGISTER_KERNEL_BUILDER(Name("NcclReduceRecv").Device(DEVICE_GPU), +REGISTER_KERNEL_BUILDER(Name("_NcclReduceRecv").Device(DEVICE_GPU), NcclReduceRecvKernel); // To execute a single broadcast, this kernel is called once for one device, and @@ -191,7 +190,7 @@ class NcclBroadcastSendKernel : public NcclAsyncOpBase { std::move(actual_done)); } }; -REGISTER_KERNEL_BUILDER(Name("NcclBroadcastSend").Device(DEVICE_GPU), +REGISTER_KERNEL_BUILDER(Name("_NcclBroadcastSend").Device(DEVICE_GPU), NcclBroadcastSendKernel); // To execute a single broadcast, this kernel is called once for all but one of @@ -206,7 +205,7 @@ class NcclBroadcastRecvKernel : public NcclAsyncOpBase { const Tensor& shape_t = c->input(0); TensorShape shape; OP_REQUIRES_OK_ASYNC( - c, TensorShapeUtils::MakeShape(shape_t.vec(), &shape), done); + c, TensorShapeUtils::MakeShape(shape_t.vec(), &shape), done); Tensor* out_t; OP_REQUIRES_OK_ASYNC(c, c->allocate_output(0, shape, &out_t), done); @@ -224,9 +223,24 @@ class NcclBroadcastRecvKernel : public NcclAsyncOpBase { } }; REGISTER_KERNEL_BUILDER( - Name("NcclBroadcastRecv").Device(DEVICE_GPU).HostMemory("shape"), + Name("_NcclBroadcastRecv").Device(DEVICE_GPU).HostMemory("shape"), NcclBroadcastRecvKernel); +// Define stub kernels for the ops that get replaced post placement. +class NcclStubKernel : public AsyncOpKernel { + public: + explicit NcclStubKernel(OpKernelConstruction* c) : AsyncOpKernel(c) {} + void ComputeAsync(OpKernelContext* c, DoneCallback done) override { + c->SetStatus(errors::Unimplemented( + "This op should be replaced during graph optimization.")); + done(); + } +}; +REGISTER_KERNEL_BUILDER(Name("NcclBroadcast").Device(DEVICE_GPU), + NcclStubKernel); +REGISTER_KERNEL_BUILDER(Name("NcclReduce").Device(DEVICE_GPU), NcclStubKernel); + +} // namespace } // namespace tensorflow #endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc new file mode 100644 index 0000000000..94a77c59da --- /dev/null +++ b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc @@ -0,0 +1,271 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#include +#include + +#include "tensorflow/core/common_runtime/optimization_registry.h" +#include "tensorflow/core/graph/node_builder.h" + +namespace tensorflow { +namespace { + +// Replaces NcclReduce node with _NcclReduceRecv reusing one input of same +// device, adds one _NcclReduceSend for each other input. +Status ReplaceReduce(Graph* graph, Node* node) { + string reduction; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "reduction", &reduction)); + DataType dtype; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype)); + int num_devices = node->num_inputs(); + string shared_name = node->name(); + auto make_builder = [&](StringPiece op_name, StringPiece suffix) { + return NodeBuilder(strings::StrCat(shared_name, suffix), op_name) + .Attr("reduction", reduction) + .Attr("num_devices", num_devices) + .Attr("shared_name", shared_name) + .Attr("T", dtype); + }; + std::vector control_inputs; + for (const auto& edge : node->in_edges()) { + if (edge->IsControlEdge()) { + control_inputs.push_back(edge->src()); + } + } + std::vector out_nodes; + for (const auto& edge : node->out_edges()) { + out_nodes.emplace_back(edge->dst(), edge->dst_input()); + } + int recv_dev = node->assigned_device_name_index(); + NodeBuilder recv_builder = + make_builder("_NcclReduceRecv", "Recv").ControlInputs(control_inputs); + bool recv_input_set = false; + int send_counter = 0; + for (const auto& edge : node->in_edges()) { + Node* src_node = edge->src(); + if (edge->IsControlEdge()) { + continue; + } + int send_dev = src_node->assigned_device_name_index(); + if (!recv_input_set && send_dev == recv_dev) { + recv_builder.Input(src_node); + recv_input_set = true; + continue; + } + auto send_builder = make_builder("_NcclReduceSend", + strings::StrCat("Send_", ++send_counter)) + .Input(src_node) + .ControlInputs(control_inputs); + Node* send_node = nullptr; + TF_RETURN_IF_ERROR(send_builder.Finalize(graph, &send_node)); + send_node->set_assigned_device_name_index(send_dev); + // Send nodes don't have any outputs and therefore have no data dependencies + // to the outputs of the graph. We add a control dependency to the receive + // node so that those 'dangling' nodes are run. + // TODO(b/67027412): Avoid these cross-device control edges. + for (const auto& out_node : out_nodes) { + graph->AddControlEdge(send_node, out_node.node); + } + } + if (!recv_input_set) { + return errors::InvalidArgument( + "No input tensor uses the same device as the NcclReduce op"); + } + Node* recv_node = nullptr; + TF_RETURN_IF_ERROR(recv_builder.Finalize(graph, &recv_node)); + recv_node->set_assigned_device_name_index(recv_dev); + graph->RemoveNode(node); + for (const auto& out_node : out_nodes) { + if (out_node.index == Graph::kControlSlot) { + graph->AddControlEdge(recv_node, out_node.node); + } else { + graph->AddEdge(recv_node, 0, out_node.node, out_node.index); + } + } + return Status::OK(); +} + +TensorProto TensorFromShape(const TensorShapeProto& shape) { + TensorProto result; + result.set_dtype(DT_INT32); + for (const auto& dim : shape.dim()) { + result.add_int_val(dim.size()); + } + result.mutable_tensor_shape()->add_dim()->set_size(shape.dim_size()); + return result; +} + +// Replaces NcclBroadcast node with _NcclBroadcastSend, connects the input to +// all outputs of same device, adds one _NcclBroadcastRecv for each other output +// device. +Status ReplaceBroadcast(Graph* graph, Node* node) { + DataType dtype; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype)); + int send_dev = node->assigned_device_name_index(); + int num_devices = 0; // Number of distinct devices, incremented below. + + // Map device name index to nodes that take the broadcast as input. + std::vector> out_nodes_map; + for (const auto& edge : node->out_edges()) { + int dst_dev = edge->IsControlEdge() + ? send_dev + : edge->dst()->assigned_device_name_index(); + if (out_nodes_map.size() <= dst_dev) { + out_nodes_map.resize(dst_dev + 1); + } + auto it = out_nodes_map.begin() + dst_dev; + if (it->empty()) { + ++num_devices; + } + it->emplace_front(NodeBuilder::NodeOut(edge->dst(), edge->dst_input())); + } + + if (num_devices <= 1) { + // Only one participating device, skip NCCL op. + const Edge* in_edge = nullptr; + TF_RETURN_IF_ERROR(node->input_edge(0, &in_edge)); + Node* in_node = in_edge->src(); + int in_index = in_edge->src_output(); + graph->RemoveNode(node); + for (const auto& out_nodes : out_nodes_map) { + for (const auto& out_node : out_nodes) { + if (out_node.index == Graph::kControlSlot) { + graph->AddControlEdge(in_node, out_node.node); + } else { + graph->AddEdge(in_node, in_index, out_node.node, out_node.index); + } + } + } + return Status::OK(); + } + + string shared_name = node->name(); + auto make_builder = [&](StringPiece op_name, StringPiece suffix) { + return NodeBuilder(strings::StrCat(shared_name, suffix), op_name) + .Attr("num_devices", num_devices) + .Attr("shared_name", shared_name) + .Attr("T", dtype); + }; + + // Create broadcast send node and replace the original broadcast node. + NodeBuilder::NodeOut in_node; + NodeBuilder send_builder = make_builder("_NcclBroadcastSend", "Send"); + for (const auto& edge : node->in_edges()) { + if (edge->IsControlEdge()) { + send_builder.ControlInput(edge->src()); + } else { + in_node = NodeBuilder::NodeOut(edge->src(), edge->src_output()); + send_builder.Input(in_node); + } + } + Node* send_node = nullptr; + TF_RETURN_IF_ERROR(send_builder.Finalize(graph, &send_node)); + send_node->set_assigned_device_name_index(send_dev); + + TensorShapeProto shape_proto; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "shape", &shape_proto)); + + // Delete the original node before reconnecting to outputs. + graph->RemoveNode(node); + + // Connect all outputs on the device of broadcast send. + for (const auto& out_node : out_nodes_map[send_dev]) { + if (out_node.index == Graph::kControlSlot) { + graph->AddControlEdge(send_node, out_node.node); + } else { + graph->AddEdge(in_node.node, in_node.index, out_node.node, + out_node.index); + // Add control edge so send node is run. + graph->AddControlEdge(send_node, out_node.node); + } + } + out_nodes_map[send_dev].clear(); + + TensorProto tensor_proto = TensorFromShape(shape_proto); + bool is_fully_defined = TensorShape(shape_proto).IsFullyDefined(); + string shape_name = strings::StrCat(in_node.node->name(), "/Shape"); + Node* shape_node = nullptr; + if (!is_fully_defined) { + NodeBuilder shape_builder(shape_name, "Shape"); + shape_builder.Input(in_node).Attr("out_type", DT_INT32).Attr("T", dtype); + TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node)); + shape_node->set_assigned_device_name_index(send_dev); + } + + // For all other devices, create a broadcast receive and connect outputs. + for (int recv_dev = 0; recv_dev < out_nodes_map.size(); ++recv_dev) { + if (out_nodes_map[recv_dev].empty()) { + continue; + } + if (is_fully_defined) { + // If the shape is fully defined, define one const node per device. + NodeBuilder shape_builder(strings::StrCat(shape_name, recv_dev), "Const"); + shape_builder.Attr("value", tensor_proto).Attr("dtype", DT_INT32); + TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node)); + shape_node->set_assigned_device_name_index(recv_dev); + } + Node* recv_node; + TF_RETURN_IF_ERROR( + make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_dev)) + .Input(shape_node) + .Finalize(graph, &recv_node)); + recv_node->set_assigned_device_name_index(recv_dev); + for (const auto& out_node : out_nodes_map[recv_dev]) { + graph->AddEdge(recv_node, 0, out_node.node, out_node.index); + } + } + + return Status::OK(); +} + +// Replaces occurrences of Nccl{Reduce, Broadcast}Input/Output with their +// _Nccl...Send/Recv counterparts and removes data dependencies between them. +class NcclReplacePass : public GraphOptimizationPass { + public: + Status Run(const GraphOptimizationPassOptions& options) override { + if (options.graph == nullptr) { + return Status::OK(); + } + Graph* graph = options.graph->get(); + if (graph == nullptr) { + return errors::Internal( + "NCCL replacement should happen before partitioning and a " + "graph should be available."); + } + // Find reduction and broadcast ops and replace them with Send/Recv ops. + for (Node* node : graph->op_nodes()) { + StringPiece type = node->type_string(); + if (!type.starts_with("Nccl")) { + continue; + } + if (type == "NcclReduce") { + TF_RETURN_IF_ERROR(ReplaceReduce(graph, node)); + } + if (type == "NcclBroadcast") { + TF_RETURN_IF_ERROR(ReplaceBroadcast(graph, node)); + } + } + return Status::OK(); + } +}; +REGISTER_OPTIMIZATION(OptimizationPassRegistry::POST_PLACEMENT, 0, + NcclReplacePass); + +} // namespace +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/nccl/ops/nccl_ops.cc b/tensorflow/contrib/nccl/ops/nccl_ops.cc index 532c79c24c..8eb804c2e9 100644 --- a/tensorflow/contrib/nccl/ops/nccl_ops.cc +++ b/tensorflow/contrib/nccl/ops/nccl_ops.cc @@ -45,7 +45,28 @@ num_devices: The number of devices participating in this reduction. shared_name: Identifier that shared between ops of the same reduction. )doc"); -REGISTER_OP("NcclReduceSend") +// Note: This op has no kernel implementation, but is replaced by +// _NcclReduceSend and _NcclReduceRecv during graph optimization stage. +REGISTER_OP("NcclReduce") + .Input("input: num_devices * T") + .Output("data: T") + .Attr("reduction: {'min', 'max', 'prod', 'sum'}") + .Attr("T: {float, float64, int32, int64}") + .Attr("num_devices: int") + .SetIsStateful() + .SetShapeFn(shape_inference::UnchangedShape) + .Doc(R"doc( +Reduces `input` from `num_devices` using `reduction` to a single device. + +The graph should be constructed so that all inputs have a valid device +assignment, and the op itself is assigned one of these devices. + +input: The input to the reduction. +data: the value of the reduction across all `num_devices` devices. +reduction: the reduction operation to perform. + )doc"); + +REGISTER_OP("_NcclReduceSend") .Input("input: T") .Attr("reduction: {'min', 'max', 'prod', 'sum'}") .Attr("T: {float, float64, int32, int64}") @@ -54,19 +75,20 @@ REGISTER_OP("NcclReduceSend") .SetIsStateful() .SetShapeFn(shape_inference::NoOutputs) .Doc(R"doc( -Reduces `input` to the NcclReduceRecv op registered in the same `shared_name`. +Replacement node for NcclReduce. +Reduces `input` to the NcclReduceRecv op registered in the same `shared_name`. The graph should be constructed so that 'num_devices-1' devices run -`NcclReduceSend` and one device runs NcclReduceRecv op with shared_name value +`_NcclReduceSend` and one device runs _NcclReduceRecv op with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. -input: The input to the reduction +input: The input to the reduction. reduction: the reduction operation to perform. num_devices: The number of devices participating in this reduction. shared_name: Identifier that is shared between ops of the same reduce. )doc"); -REGISTER_OP("NcclReduceRecv") +REGISTER_OP("_NcclReduceRecv") .Input("input: T") .Output("data: T") .Attr("reduction: {'min', 'max', 'prod', 'sum'}") @@ -76,21 +98,42 @@ REGISTER_OP("NcclReduceRecv") .SetIsStateful() .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( +Replacement node for NcclReduce. + Reduces 'input' from this op and the NcclReduceSend ops registered in the same `shared_name`. - The graph should be constructed so that 'num_devices-1' devices run -`NcclReduceSend` and one device runs NcclReduceRecv op with shared_name value +`_NcclReduceSend` and one device runs _NcclReduceRecv op with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. -input: The input to the reduction +input: The input to the reduction. data: The reduced data received from this op and the NcclReduceSend op. reduction: the reduction operation to perform. num_devices: The number of devices participating in this reduction. shared_name: Identifier that is shared between ops of the same reduce. )doc"); -REGISTER_OP("NcclBroadcastSend") +// Note: This op has no kernel implementation, but is replaced by +// _NcclBroadcastSend and _NcclBroadcastRecv during graph optimization stage. +REGISTER_OP("NcclBroadcast") + .Input("input: T") + .Output("output: T") + .Attr("T: {float, float64, int32, int64}") + .Attr("shape: shape") + .SetIsStateful() + .SetShapeFn(shape_inference::UnchangedShape) + .Doc(R"doc( +Sends `input` to all devices that are connected to the output. + +The graph should be constructed so that all ops connected to the output have a +valid device assignment, and the op itself is assigned one of these devices. + +input: The input to the broadcast. +output: The same as input. +shape: The shape of the input tensor. + )doc"); + +REGISTER_OP("_NcclBroadcastSend") .Input("input: T") .Attr("T: {float, float64, int32, int64}") .Attr("num_devices: int") @@ -98,19 +141,21 @@ REGISTER_OP("NcclBroadcastSend") .SetIsStateful() .SetShapeFn(shape_inference::NoOutputs) .Doc(R"doc( -Sends `input` to the NcclBroadcastRecv ops registered in the same `shared_name`. +Replacement node for NcclBroadcast. -The graph should be constructed so that one device runs `NcclBroadcastSend` and -`num_devices-1` devices run NcclBroadcastRecv ops with shared_name value `c`. +Sends `input` to the _NcclBroadcastRecv ops registered in the same +`shared_name`. +The graph should be constructed so that one device runs `_NcclBroadcastSend` and +`num_devices-1` devices run _NcclBroadcastRecv ops with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. -input: The input to the broadcast +input: The input to the broadcast. num_devices: The number of devices participating in this reduction. shared_name: Identifier that is shared between ops of the same broadcast. )doc"); -REGISTER_OP("NcclBroadcastRecv") - .Input("shape: int64") +REGISTER_OP("_NcclBroadcastRecv") + .Input("shape: int32") .Output("output: T") .Attr("T: {float, float64, int32, int64}") .Attr("num_devices: int") @@ -123,11 +168,12 @@ REGISTER_OP("NcclBroadcastRecv") return Status::OK(); }) .Doc(R"doc( -Sends data of shape `shape` from the NcclBroadcastSend op registered in the -same `shared_name`. +Replacement node for NcclBroadcast. -The graph should be constructed so that one device runs `NcclBroadcastSend` and -`num_devices-1` devices run NcclBroadcastRecv ops with shared_name value `c`. +Sends data of shape `shape` from the _NcclBroadcastSend op registered in the +same `shared_name`. +The graph should be constructed so that one device runs `_NcclBroadcastSend` and +`num_devices-1` devices run _NcclBroadcastRecv ops with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. shape: The shape of the output. diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops.py b/tensorflow/contrib/nccl/python/ops/nccl_ops.py index 906d9f948a..8dc038b9ac 100644 --- a/tensorflow/contrib/nccl/python/ops/nccl_ops.py +++ b/tensorflow/contrib/nccl/python/ops/nccl_ops.py @@ -23,9 +23,7 @@ from tensorflow.contrib.nccl.ops import gen_nccl_ops from tensorflow.contrib.util import loader from tensorflow.python.eager import context from tensorflow.python.framework import device -from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops from tensorflow.python.platform import resource_loader _nccl_ops_so = loader.load_op_library( @@ -64,13 +62,13 @@ def _all_sum_grad(op, grad): LookupError: If `reduction` is not `sum`. """ if op.get_attr('reduction') != 'sum': - raise LookupError('No gradient defined for NcclAllReduce except all_sum.') + raise LookupError('No gradient defined for NcclAllReduce except sum.') - _check_device_assignment(grad) + _check_device(grad, expected=op.device) num_devices = op.get_attr('num_devices') shared_name = op.get_attr('shared_name') + '_grad' - with ops.device(grad.device): + with ops.device(op.device): return gen_nccl_ops.nccl_all_reduce( input=grad, reduction='sum', @@ -129,7 +127,7 @@ def all_max(tensors): return _apply_all_reduce('max', tensors) -def reduce_sum(tensors, dst_device): +def reduce_sum(tensors): """Returns a tensor with the reduce sum across `tensors`. The computation is done with a reduce operation, so only one tensor is @@ -138,54 +136,76 @@ def reduce_sum(tensors, dst_device): Args: tensors: The input tensors across which to sum; must be assigned to GPU devices. - dst_device: The device of the returned tensor. Returns: - A tensor containing the sum of the input tensors, with the device of the - tensor being `dst_device`. + A tensor containing the sum of the input tensors. + + Raises: + LookupError: If context is not currently using a GPU device. + """ + return _apply_reduce('sum', tensors) + + +@ops.RegisterGradient('NcclReduce') +def _reduce_sum_grad(op, grad): + """The gradients for input `Operation` of `reduce_sum`. + + Args: + op: The `sum send` `Operation` that we are differentiating. + grad: Gradient with respect to the output of the `reduce_sum` op. + + Returns: + The gradient with respect to the input of `reduce_sum` op. + + Raises: + LookupError: If the reduction attribute of op is not `sum`. """ - return _apply_reduce('sum', tensors, dst_device) + if op.get_attr('reduction') != 'sum': + raise LookupError('No gradient defined for NcclReduce except sum.') + _check_device(grad, expected=op.device) + with ops.device(op.device): + result = gen_nccl_ops.nccl_broadcast(input=grad, shape=grad.shape) -def broadcast(src_tensor, dst_devices): - """Returns a list of tensors on `dst_devices`, each with value `tensor`. + return [result] * len(op.inputs) - The computation is done with a broadcast nccl operation, so if only some of - the returned tensors and src_tensor are evaluated then the computation will - hang. + +def broadcast(tensor): + """Returns a tensor that can be efficiently transferred to other devices. Args: - src_tensor: The tensor to send; must be assigned to a GPU device. - dst_devices: The GPU devices to receive the sent tensor. + tensor: The tensor to send; must be assigned to a GPU device. Returns: - An `Operation` to send the `src_tensor`, and a list of tensors, each with - the value of `src_tensor`, where the device of tensor i is `dst_devices[i]`. + A tensor with the value of `src_tensor`, which can be used as input to + ops on other GPU devices. """ - if not dst_devices: - raise ValueError('Must pass >0 dst_devices to broadcast') _check_graph_mode() - _check_device_assignment(src_tensor) + _check_device(tensor) - shape = array_ops.shape(src_tensor, out_type=dtypes.int64) - num_devices = len(dst_devices) + 1 - shared_name = _get_shared_name() + with ops.device(tensor.device): + return gen_nccl_ops.nccl_broadcast(input=tensor, shape=tensor.shape) - with ops.device(src_tensor.device): - send = gen_nccl_ops.nccl_broadcast_send( - input=src_tensor, num_devices=num_devices, shared_name=shared_name) - - recvs = [] - for d in dst_devices: - with ops.device(d): - recvs.append( - gen_nccl_ops.nccl_broadcast_recv( - shape=shape, - T=src_tensor.dtype, - num_devices=num_devices, - shared_name=shared_name)) - return send, recvs +@ops.RegisterGradient('NcclBroadcast') +def _broadcast_grad(op, accumulated_grad): + """The gradients for input `Operation` of `broadcast`. + + Args: + op: The `broadcast send` `Operation` that we are differentiating. + accumulated_grad: Accumulated gradients with respect to the output of the + `broadcast` op. + + Returns: + Gradients with respect to the input of `broadcast`. + """ + # Grab inputs of accumulated_grad and replace accumulation with reduce_sum. + grads = [t for t in accumulated_grad.op.inputs] + for t in grads: + _check_device(t) + + with ops.device(op.device): + return gen_nccl_ops.nccl_reduce(input=grads, reduction='sum') def _apply_all_reduce(reduction, tensors): @@ -198,7 +218,7 @@ def _apply_all_reduce(reduction, tensors): res = [] for t in tensors: - _check_device_assignment(t) + _check_device(t) with ops.device(t.device): res.append( gen_nccl_ops.nccl_all_reduce( @@ -210,40 +230,20 @@ def _apply_all_reduce(reduction, tensors): return res -def _apply_reduce(reduction, tensors, dst_device): +def _apply_reduce(reduction, tensors): """Helper function for reduce_* functions.""" if not tensors: raise ValueError('Must pass >0 tensors to reduce operations') - if not dst_device: - raise ValueError('Must pass dst_device to reduce operations') _check_graph_mode() + for t in tensors: + _check_device(t) + result = gen_nccl_ops.nccl_reduce(input=tensors, reduction=reduction) try: - recv_index = next(i for i, t in enumerate(tensors) - if t.device == dst_device) + next(t for t in tensors if t.device == result.device) except StopIteration: - raise ValueError('One of the tensors must be assigned to dst_device') - shared_name = _get_shared_name() - - sends = [] - for t in tensors[:recv_index] + tensors[recv_index + 1:]: - _check_device_assignment(t) - with ops.device(t.device): - sends.append( - gen_nccl_ops.nccl_reduce_send( - input=t, - reduction=reduction, - num_devices=len(tensors), - shared_name=shared_name)) - - with ops.device(dst_device): - recv = gen_nccl_ops.nccl_reduce_recv( - input=tensors[recv_index], - reduction=reduction, - num_devices=len(tensors), - shared_name=shared_name) - - return recv, sends + raise ValueError('One input tensor must be assigned to current device') + return result _lock = threading.Lock() @@ -259,9 +259,11 @@ def _get_shared_name(): return 'c%s' % val -def _check_device_assignment(tensor): +def _check_device(tensor, expected=None): if not device.canonical_name(tensor.device): raise ValueError('Device assignment required for nccl collective ops') + if expected and expected != tensor.device: + raise ValueError('Expected device %s, got %s' % (expected, tensor.device)) def _check_graph_mode(): diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py index 96d67723a0..255409303a 100644 --- a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py +++ b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py @@ -22,8 +22,10 @@ from functools import partial import numpy as np from tensorflow.contrib import nccl +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradients from tensorflow.python.platform import test @@ -36,27 +38,30 @@ def _DeviceTensors(tensors, devices): def _NcclAllReduce(nccl_fun, tensors, devices): - return nccl_fun(_DeviceTensors(tensors, devices)), [] + return nccl_fun(_DeviceTensors(tensors, devices)) def _NcclReduce(nccl_fun, tensors, devices): - d_tensors = _DeviceTensors(tensors, devices) receiver = np.random.randint(0, len(devices)) - received_tensor, send_ops = nccl_fun(d_tensors, devices[receiver]) - return [received_tensor], send_ops + with ops.device(devices[receiver]): + return [nccl_fun(_DeviceTensors(tensors, devices))] def _NcclBroadcast(tensors, devices): sender = np.random.randint(0, len(devices)) - d_tensor = _DeviceTensors(tensors[0:1], devices[sender:sender + 1])[0] - other_devices = devices[:sender] + devices[sender + 1:] - send_op, received_tensors = nccl.broadcast(d_tensor, other_devices) - return received_tensors, [send_op] + with ops.device(devices[sender]): + tensor = array_ops.identity(tensors[0]) + broadcast = nccl.broadcast(tensor) + return _DeviceTensors([broadcast] * len(devices), devices) class NcclTestCase(test.TestCase): - def _Test(self, nccl_reduce, numpy_fn): + def _Test(self, + nccl_reduce, + numpy_fn, + device_sets=(['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'], + ['/device:GPU:1', '/device:GPU:0'])): """Tests that nccl_reduce does the same as reduction with numpy_fn. Args: @@ -65,6 +70,7 @@ class NcclTestCase(test.TestCase): reduction. numpy_fn: A function taking two tensors and returning the reduction of the two. + device_sets: Tuple of virtual devices to run test on. """ if not test.is_gpu_available(): return # Test requires access to a GPU @@ -74,26 +80,28 @@ class NcclTestCase(test.TestCase): # same communicator across multiple sessions. with self.test_session(use_gpu=True) as sess: - for devices in [['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'], - ['/device:GPU:1', '/device:GPU:0']]: + for devices in device_sets: shape = (3, 4) random = (np.random.random_sample(shape) - .5) * 1024 - tensors = [random.astype(dtype)] * len(devices) + tensors = [] + for _ in devices: + tensors.append(random.astype(dtype)) np_ans = tensors[0] for t in tensors[1:]: np_ans = numpy_fn(np_ans, t) - reduce_tensors, reduce_ops = nccl_reduce(tensors, devices) + reduce_tensors = nccl_reduce(tensors, devices) self.assertNotEmpty(reduce_tensors) # Test shape inference. for r in reduce_tensors: self.assertEqual(shape, r.get_shape()) + result_tensors = [array_ops.identity(t) for t in reduce_tensors] + # Test execution and results. - nccl_results = sess.run(reduce_tensors + reduce_ops) - for r in nccl_results[:len(reduce_tensors)]: - self.assertAllClose(r, np_ans) + for t in sess.run(result_tensors): + self.assertAllClose(t, np_ans) def _TestGradient(self, nccl_reduce, numpy_fn): """Tests the gradient of nccl_reduce. @@ -106,14 +114,11 @@ class NcclTestCase(test.TestCase): reduction of the two. """ def _Gradient(tensors, devices): - reduce_tensors, _ = nccl_reduce(tensors, devices) - tensor_ops = [t.op for t in reduce_tensors] - d_tensors = _DeviceTensors(tensors, devices) - grad_tensors = [ - ops.get_gradient_function(op)(op, loss) - for op, loss in zip(tensor_ops, d_tensors) - ] - return grad_tensors, [] + inputs = [array_ops.placeholder(t.dtype, t.shape) for t in tensors] + reduce_tensors = nccl_reduce(inputs, devices) + losses = _DeviceTensors(tensors, [t.device for t in reduce_tensors]) + grads = gradients.gradients(reduce_tensors, inputs, losses) + return [g for g in grads if g is not None] self._Test(_Gradient, numpy_fn) @@ -142,27 +147,43 @@ class SingleReduceTest(NcclTestCase): def testSum(self): self._Test(partial(_NcclReduce, nccl.reduce_sum), lambda x, y: x + y) + def testSumGrad(self): + self._TestGradient(partial(_NcclReduce, nccl.reduce_sum), lambda x, y: x) + class BroadcastTest(NcclTestCase): def testBroadcast(self): self._Test(_NcclBroadcast, lambda x, y: x) + def testBroadcastSingleDevice(self): + # Broadcasts on a single device are removed completely during rewrite. + self._Test(_NcclBroadcast, lambda x, y: x, + (['/device:GPU:0', '/device:GPU:0'])) + + def testBroadcastToCpuError(self): + # Broadcasts to CPU is not supported. + with self.assertRaisesRegexp( + errors.NotFoundError, + "No registered '_NcclBroadcastRecv' OpKernel for CPU devices"): + self._Test(_NcclBroadcast, lambda x, y: x, + (['/device:GPU:0', '/device:CPU:0'])) + + def testBroadcastGrad(self): + self._TestGradient(_NcclBroadcast, lambda x, y: x + y) + class CombinedTest(NcclTestCase): """Test all-reduce vs. single-reduce plus broadcast in one session.run.""" - def _combined(self, tensors, devices): - all_reduce_tensors = _NcclAllReduce(nccl.all_sum, tensors, devices)[0] - single_reduce_tensors, single_reduce_ops = _NcclReduce( - nccl.reduce_sum, tensors, devices) - broadcast_tensors, broadcast_ops = _NcclBroadcast(single_reduce_tensors, - devices) - all_tensors = all_reduce_tensors + single_reduce_tensors + broadcast_tensors - return all_tensors, single_reduce_ops + broadcast_ops + def _Combined(self, tensors, devices): + all_reduce_tensors = _NcclAllReduce(nccl.all_sum, tensors, devices) + single_reduce_tensors = _NcclReduce(nccl.reduce_sum, tensors, devices) + broadcast_tensors = _NcclBroadcast(single_reduce_tensors, devices) + return all_reduce_tensors + broadcast_tensors def testCombined(self): - self._Test(self._combined, lambda x, y: x + y) + self._Test(self._Combined, lambda x, y: x + y) if __name__ == '__main__': -- GitLab From 1324d40ad7996e4f2afd8f5c5b5f68416c9872e1 Mon Sep 17 00:00:00 2001 From: Anna R Date: Sun, 15 Oct 2017 22:46:47 -0700 Subject: [PATCH 003/573] Internal change. PiperOrigin-RevId: 172282778 --- tensorflow/python/estimator/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index e4b2d95acd..9670827e41 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -146,6 +146,7 @@ py_test( srcs = ["training_test.py"], shard_count = 4, srcs_version = "PY2AND3", + tags = ["notsan"], deps = [ ":dnn", ":estimator", -- GitLab From 48c4b45ab68f09317415ef2b03c74e349319ce8b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 05:51:00 -0700 Subject: [PATCH 004/573] Remove unused BUILD dependencies PiperOrigin-RevId: 172314225 --- tensorflow/compiler/xla/service/BUILD | 1 - tensorflow/compiler/xla/service/llvm_ir/BUILD | 1 - tensorflow/contrib/cloud/kernels/BUILD | 21 +++++++------------ tensorflow/core/kernels/BUILD | 1 - tensorflow/core/platform/cloud/BUILD | 17 ++++++--------- 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 3e85c796f2..d1335e20e0 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -433,7 +433,6 @@ cc_library( ":hlo_evaluator", ":hlo_execution_profile", ":hlo_module_config", - ":hlo_verifier", ":platform_util", ":session_proto", ":transfer_manager", diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD index 70579e3273..075d4a1ab5 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/BUILD +++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD @@ -137,7 +137,6 @@ cc_library( "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service/gpu:parallel_loop_emitter", "//tensorflow/compiler/xla/service/gpu:partition_assignment", - "@llvm//:core", ], ) diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD index 09ec7e42c7..56f930a9a8 100644 --- a/tensorflow/contrib/cloud/kernels/BUILD +++ b/tensorflow/contrib/cloud/kernels/BUILD @@ -23,7 +23,9 @@ load( filegroup( name = "all_files", srcs = glob( - ["**/*"], + include = [ + "**/*", + ], exclude = [ "**/METADATA", "**/OWNERS", @@ -34,9 +36,7 @@ filegroup( tf_kernel_library( name = "bigquery_reader_ops", - srcs = [ - "bigquery_reader_ops.cc", - ], + srcs = ["bigquery_reader_ops.cc"], visibility = ["//visibility:public"], deps = [ ":bigquery_table_accessor", @@ -50,12 +50,8 @@ tf_kernel_library( cc_library( name = "bigquery_table_accessor", - srcs = [ - "bigquery_table_accessor.cc", - ], - hdrs = [ - "bigquery_table_accessor.h", - ], + srcs = ["bigquery_table_accessor.cc"], + hdrs = ["bigquery_table_accessor.h"], copts = tf_copts(), linkstatic = 1, deps = [ @@ -64,7 +60,6 @@ cc_library( "//tensorflow/core:protos_all_cc", "//tensorflow/core/platform/cloud:curl_http_request", "//tensorflow/core/platform/cloud:google_auth_provider", - "//tensorflow/core/platform/cloud:http_request", ], alwayslink = 1, ) @@ -88,8 +83,6 @@ tf_cc_test( tf_proto_library( name = "bigquery_table_partition_proto", - srcs = [ - "bigquery_table_partition.proto", - ], + srcs = ["bigquery_table_partition.proto"], cc_api_version = 2, ) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 2c02571346..d1a2362e5e 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -5671,7 +5671,6 @@ cc_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/util/tensor_bundle", ], ) diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD index c937fea049..901fb79d6a 100644 --- a/tensorflow/core/platform/cloud/BUILD +++ b/tensorflow/core/platform/cloud/BUILD @@ -15,7 +15,9 @@ load( filegroup( name = "all_files", srcs = glob( - ["**/*"], + include = [ + "**/*", + ], exclude = [ "**/METADATA", "**/OWNERS", @@ -41,12 +43,8 @@ cc_library( cc_library( name = "gcs_file_system", - srcs = [ - "gcs_file_system.cc", - ], - hdrs = [ - "gcs_file_system.h", - ], + srcs = ["gcs_file_system.cc"], + hdrs = ["gcs_file_system.h"], linkstatic = 1, # Needed since alwayslink is broken in bazel b/27630669 visibility = ["//visibility:public"], deps = [ @@ -106,9 +104,7 @@ cc_library( cc_library( name = "google_auth_provider", - srcs = [ - "google_auth_provider.cc", - ], + srcs = ["google_auth_provider.cc"], hdrs = [ "auth_provider.h", "google_auth_provider.h", @@ -116,7 +112,6 @@ cc_library( visibility = ["//tensorflow:__subpackages__"], deps = [ ":curl_http_request", - ":http_request", ":oauth_client", ":retrying_utils", "//tensorflow/core:lib", -- GitLab From 610e1185da37ce8415db64ae60150f253b3fadc9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 06:07:14 -0700 Subject: [PATCH 005/573] - Modified Jacobian computations in CurvatureMatrixVectorProductComputer to use true partial derivatives. This is done using the newly introduced stop_gradients argument to tf.gradients. PiperOrigin-RevId: 172315620 --- .../ops/curvature_matrix_vector_products.py | 26 +++++++++---------- tensorflow/contrib/kfac/python/ops/utils.py | 5 ++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py b/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py index bf59a92fa6..21b5cde9b9 100644 --- a/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py +++ b/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py @@ -36,13 +36,13 @@ class CurvatureMatrixVectorProductComputer(object): For example, the Fisher associated with a log-prob loss w.r.t. the parameters. - The vecs argument to each method are lists of tensors that must be the + The 'vecs' argument to each method are lists of tensors that must be the size as the corresponding ones from "wrt_tensors". They represent the vector being multiplied. "factors" of the matrix M are defined as matrices B such that B*B^T = M. - Methods that multiply by the factor B take a "loss_inner_vecs" argument - instead of vecs, which must be a list of tensors with shapes given by the + Methods that multiply by the factor B take a 'loss_inner_vecs' argument + instead of 'vecs', which must be a list of tensors with shapes given by the corresponding XXX_inner_shapes property. Note that matrix-vector products are not normalized by the batch size, nor @@ -61,7 +61,8 @@ class CurvatureMatrixVectorProductComputer(object): Args: losses: A list of LossFunction instances whose sum defines the total loss. wrt_tensors: A list of Tensors to compute the differential quantities - defining the matrices with respect to (see class description). + (defining the matrices) with respect to. See class description for more + info. """ self._losses = losses self._inputs_to_losses = list(loss.inputs for loss in losses) @@ -73,24 +74,23 @@ class CurvatureMatrixVectorProductComputer(object): return math_ops.add_n(tuple(loss.evaluate() for loss in self._losses)) # Jacobian multiplication functions: - # NOTE: These implementations use tf.gradients and thus aren't actually - # computing partial derivatives, but total derivatives instead (despite what - # the documentation for tf.gradients says). Because we require partial - # derivatives for Jacobians this implementation will only be correct if the - # partial derivatives are equal to the full derivatives. This happens as long - # as the elements of wrt_tensors don't depend on each other in the graph. If - # these tensors are standard neural network parameters this will be true. def _multiply_jacobian(self, vecs): """Multiply vecs by the Jacobian of losses.""" + # We stop gradients at wrt_tensors to produce partial derivatives (which is + # what we want for Jacobians). jacobian_vecs_flat = utils.fwd_gradients( - self._inputs_to_losses_flat, self._wrt_tensors, grad_xs=vecs) + self._inputs_to_losses_flat, self._wrt_tensors, grad_xs=vecs, + stop_gradients=self._wrt_tensors) return nest.pack_sequence_as(self._inputs_to_losses, jacobian_vecs_flat) def _multiply_jacobian_transpose(self, loss_vecs): """Multiply vecs by the transpose Jacobian of losses.""" loss_vecs_flat = nest.flatten(loss_vecs) + # We stop gradients at wrt_tensors to produce partial derivatives (which is + # what we want for Jacobians). return gradients_impl.gradients( - self._inputs_to_losses_flat, self._wrt_tensors, grad_ys=loss_vecs_flat) + self._inputs_to_losses_flat, self._wrt_tensors, grad_ys=loss_vecs_flat, + stop_gradients=self._wrt_tensors) # Losses Fisher/Hessian multiplication functions: def _multiply_loss_fisher(self, loss_vecs): diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py index b34b4e10ad..a7473481e4 100644 --- a/tensorflow/contrib/kfac/python/ops/utils.py +++ b/tensorflow/contrib/kfac/python/ops/utils.py @@ -250,7 +250,7 @@ def generate_random_signs(shape, dtype=dtypes.float32): return 2 * math_ops.cast(ints, dtype=dtype) - 1 -def fwd_gradients(ys, xs, grad_xs=None): +def fwd_gradients(ys, xs, grad_xs=None, stop_gradients=None): """Compute forward-mode gradients.""" # See b/37888268. @@ -260,7 +260,8 @@ def fwd_gradients(ys, xs, grad_xs=None): # generated by the first gradients_impl.gradients call. us = [array_ops.zeros_like(y) + float("nan") for y in ys] - dydxs = gradients_impl.gradients(ys, xs, grad_ys=us) + dydxs = gradients_impl.gradients(ys, xs, grad_ys=us, + stop_gradients=stop_gradients) # Deal with strange types that gradients_impl.gradients returns but can't # deal with. -- GitLab From 28f0b54ad0009210a3734486f88804134f34abd0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 07:13:21 -0700 Subject: [PATCH 006/573] PiperOrigin-RevId: 172320984 --- tensorflow/core/distributed_runtime/scheduler.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/core/distributed_runtime/scheduler.cc b/tensorflow/core/distributed_runtime/scheduler.cc index 844a0643e6..4766f4c33b 100644 --- a/tensorflow/core/distributed_runtime/scheduler.cc +++ b/tensorflow/core/distributed_runtime/scheduler.cc @@ -226,7 +226,6 @@ Microseconds GreedyScheduler::ComputeSchedule( while (!event_queue.empty()) { Event event = event_queue.top(); event_queue.pop(); - Microseconds curr_time; if (event.is_completion) { Sim* sim = device_states_[event.node->assigned_device_name()]; --sim->num_running; -- GitLab From bba2bc1c6c6a47c2b9c0889d4aa2628a0cdf6c92 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 07:52:10 -0700 Subject: [PATCH 007/573] Fixing comment mismatch. PiperOrigin-RevId: 172324333 --- tensorflow/contrib/boosted_trees/examples/binary_mnist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py index 9be362f5c8..c003b1de66 100644 --- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py +++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py @@ -52,7 +52,7 @@ def get_input_fn(data, ids = np.where((data.labels == 4) | (data.labels == 9)) images = data.images[ids] labels = data.labels[ids] - # Make digit 4 label 0, 9 is 1. + # Make digit 4 label 1, 9 is 0. labels = labels == 4 def _input_fn(): -- GitLab From a799ade213cecb3c1c1d19eca6a0bfa3fddf0113 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 08:04:40 -0700 Subject: [PATCH 008/573] Automated g4 rollback of changelist 171877766 PiperOrigin-RevId: 172325692 --- tensorflow/compiler/xla/service/cpu/BUILD | 18 ++ .../compiler/xla/service/cpu/cpu_compiler.cc | 15 +- .../compiler/xla/service/cpu/cpu_compiler.h | 2 +- .../cpu/cpu_parallelization_preparation.cc | 20 -- .../compiler/xla/service/cpu/cpu_runtime.cc | 3 + .../compiler/xla/service/cpu/cpu_runtime.h | 1 + .../compiler/xla/service/cpu/ir_emitter.cc | 192 +++++++++++++++--- .../compiler/xla/service/cpu/ir_emitter.h | 21 +- .../service/cpu/parallel_task_assignment.cc | 148 ++++++++++++-- .../service/cpu/parallel_task_assignment.h | 49 +++++ .../xla/service/cpu/runtime_fork_join.cc | 97 +++++++++ .../xla/service/cpu/runtime_fork_join.h | 33 +++ .../xla/service/cpu/simple_orc_jit.cc | 2 + tensorflow/compiler/xla/tests/BUILD | 2 + tensorflow/compiler/xla/tests/fusion_test.cc | 136 +++++++++++-- 15 files changed, 648 insertions(+), 91 deletions(-) create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.h diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 8ab358fe17..c71eca0d39 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -87,6 +87,7 @@ cc_library( ":ir_emitter", ":layout_assignment", ":parallel_cpu_executable", + ":parallel_task_assignment", ":simple_orc_jit", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:protobuf_util", @@ -155,6 +156,7 @@ cc_library( ":disassembler", ":external_constant_pool", ":runtime_conv2d", + ":runtime_fork_join", ":runtime_matmul", ":runtime_single_threaded_conv2d", ":runtime_single_threaded_matmul", @@ -243,6 +245,7 @@ cc_library( ":dot_op_emitter", ":external_constant_pool", ":ir_emission_utils", + ":shape_partition", ":simple_orc_jit", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -505,6 +508,20 @@ cc_library( ], ) +cc_library( + name = "runtime_fork_join", + srcs = ["runtime_fork_join.cc"], + hdrs = ["runtime_fork_join.h"], + copts = runtime_copts(), + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/xla:executable_run_options", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//third_party/eigen3", + ], +) + tf_cc_test( name = "cpu_runtime_test", srcs = ["cpu_runtime_test.cc"], @@ -688,6 +705,7 @@ cc_library( ":shape_partition", "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service:hlo_cost_analysis", + "//tensorflow/compiler/xla/service:hlo_pass", ], ) diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 1437fb4cf9..ce4d109214 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -58,6 +58,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/cpu/layout_assignment.h" #include "tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h" +#include "tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/flatten_call_graph.h" @@ -248,7 +249,7 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault { }; } // namespace -Status CpuCompiler::RunHloPasses(HloModule* module) { +Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) { // Optimization pipeline. HloPassPipeline pipeline("CPU"); pipeline.AddInvariantChecker(ShapeSizeBytesFunction()); @@ -316,6 +317,14 @@ Status CpuCompiler::RunHloPasses(HloModule* module) { if (options::CpuParallelBackendRequested(module->config())) { pipeline.AddPass(max_parallelism, ShapeSizeBytesFunction()); + } else if (!is_aot_compile) { + // Run ParallelTaskAssigner to assign parallel tasks to HLOs in module. + // Note this is not run for AOT because it would bring in thread pool + // and thread synchronization dependencies which would likely increase + // binary size (and most AOT applications are single-threaded). + // TODO(29630486) Support multi-threaded AOT. + pipeline.AddPass(max_parallelism, + ShapeSizeBytesFunction(), module); } // Copy insertion should be performed immediately before IR emission to avoid // inserting unnecessary copies (later pass adds an instruction which @@ -450,7 +459,7 @@ StatusOr> CpuCompiler::Compile( llvm_module->setDataLayout(jit->data_layout()); llvm_module->setTargetTriple(jit->target_triple().getTriple()); - TF_RETURN_IF_ERROR(RunHloPasses(module.get())); + TF_RETURN_IF_ERROR(RunHloPasses(module.get(), /*is_aot_compile=*/false)); HloComputation* computation = module->entry_computation(); std::unordered_map hlo_to_profile_idx; @@ -749,7 +758,7 @@ CpuCompiler::CompileAheadOfTime(std::vector> modules, HloModule* module = modules[i].get(); VLOG(1) << "Compiling ahead-of-time: " << module->name(); - TF_RETURN_IF_ERROR(RunHloPasses(module)); + TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true)); TF_ASSIGN_OR_RETURN( SequentialHloOrdering::HloModuleSequence module_sequence, diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h index a301d04337..d091302474 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h @@ -132,7 +132,7 @@ class CpuCompiler : public LLVMCompiler { // Runs the HLO passes which are necessary for both optimizations and // correctness. - Status RunHloPasses(HloModule* module); + Status RunHloPasses(HloModule* module, bool is_aot_compile); TF_DISALLOW_COPY_AND_ASSIGN(CpuCompiler); }; diff --git a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc index 2cd0aa7880..662ee60923 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc @@ -116,26 +116,6 @@ StatusOr ParallelizationPreparation::RunParallelTaskAssignment( // Assign parallel tasks to HLOs in entry computation. HloComputation* computation = module->entry_computation(); for (auto* instruction : computation->instructions()) { - // Currently, we do not assign parallel tasks to instructions with at least - // one of the following properties: - // *) Internal threading (library calls to kConv, kDot, and kCustomCall). - // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot). - // *) Tuple-shaped. - // TODO(b/27458679) Parallelize instructions which are skipped here. - if (instruction->opcode() == HloOpcode::kParameter || - instruction->opcode() == HloOpcode::kConstant || - instruction->opcode() == HloOpcode::kCall || - instruction->opcode() == HloOpcode::kCustomCall || - instruction->opcode() == HloOpcode::kSelectAndScatter || - (instruction->opcode() == HloOpcode::kConvolution && - PotentiallyImplementedAsEigenConvolution(*instruction)) || - PotentiallyImplementedAsEigenDot(*instruction) || - (instruction->opcode() == HloOpcode::kFusion && - instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) || - ShapeUtil::IsTuple(instruction->shape())) { - continue; - } - // Calculate target parallel task count in [1, max_parallelism_]. const int64 target_parallel_task_count = parallel_task_assignment.GetTargetParallelTaskCount(instruction); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc index c7155b858b..7908dc173d 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc @@ -51,6 +51,9 @@ extern const char* const kAcquireOutfeedBufferForPopulationSymbolName = "__xla_cpu_runtime_AcquireOutfeedBufferForPopulation"; extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName = "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation"; +extern const char* const kParallelForkJoinSymbolName = + "__xla_cpu_runtime_ParallelForkJoin"; + extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_"; } // namespace runtime } // namespace cpu diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h index 29feb7267f..2ade455b8a 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h @@ -51,6 +51,7 @@ extern const char* const kAcquireInfeedBufferForDequeueSymbolName; extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName; extern const char* const kAcquireOutfeedBufferForPopulationSymbolName; extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName; +extern const char* const kParallelForkJoinSymbolName; // All symbol names for XLA CPU runtime functions need to start with this // prefix. diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 3d2d0f1029..52085d1376 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -42,6 +42,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h" #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h" #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h" +#include "tensorflow/compiler/xla/service/cpu/shape_partition.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" @@ -186,20 +187,9 @@ void IrEmitter::InitializeIrFunction(const string& function_name) { // Even though the type of params and temps is void** in the host's view, in // LLVM IR this is represented by i8*, similarly to void*. It's up to the code // to use GEPs to unravel the indirection layers. - llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext()); - llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo(); - llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext()); - std::vector compute_function_params( - {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type}); - if (IsParallelContext()) { - compute_function_params.push_back(i64_ptr_type); - } - if (hlo_to_profile_idx_) { - compute_function_params.push_back(i64_ptr_type); - } llvm::FunctionType* compute_function_type = llvm::FunctionType::get( /*Result=*/llvm::Type::getVoidTy(module_->getContext()), - /*Params=*/compute_function_params, + /*Params=*/GetComputeFunctionParams(), /*isVarArg=*/false); // Functions with local linkage get an inlining bonus. Because we know @@ -221,7 +211,7 @@ void IrEmitter::InitializeIrFunction(const string& function_name) { (++arg_iter)->setName("run_options"); (++arg_iter)->setName("params"); (++arg_iter)->setName("temps"); - if (IsParallelContext()) { + if (num_dynamic_loop_bounds_ > 0) { (++arg_iter)->setName("dynamic_loop_bounds"); } if (hlo_to_profile_idx_) { @@ -2288,8 +2278,19 @@ Status IrEmitter::HandleCall(HloInstruction* call) { } TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call)); - EmitArrayFunctionCallInto(call_ir_function, parameter_addresses, - emitted_value_[call], computation->name()); + + if (!computation->root_instruction()->outer_dimension_partitions().empty() && + !parallel_cpu_backend_) { + // ParallelTaskAssignment assigned partitions, emit call to + // ParallelForkJoin. + TF_RETURN_IF_ERROR(EmitParallelForkJoin(parameter_addresses, + emitted_value_[call], computation, + call_ir_function)); + } else { + EmitArrayFunctionCallInto(call_ir_function, parameter_addresses, + emitted_value_[call], computation->name()); + } + return Status::OK(); } @@ -2599,7 +2600,7 @@ Status IrEmitter::FinishVisit(HloInstruction* root) { // For the parallel cpu backend, we record the total for each embedded // computation callee with its caller kCall HLO. HloInstruction* hlo_to_lookup = nullptr; - if (IsParallelContext()) { + if (parallel_cpu_backend_ && is_top_level_computation_) { auto* computation = root->parent(); auto* entry_computation = computation->parent()->entry_computation(); if (computation != entry_computation) { @@ -2757,12 +2758,27 @@ llvm::Type* IrEmitter::IrShapeType(const Shape& shape) { return llvm_ir::ShapeToIrType(shape, &ir_builder_); } +std::vector IrEmitter::GetComputeFunctionParams() { + llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext()); + llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo(); + llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext()); + std::vector compute_function_params( + {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type}); + if (num_dynamic_loop_bounds_ > 0) { + compute_function_params.push_back(i64_ptr_type); + } + if (hlo_to_profile_idx_) { + compute_function_params.push_back(i64_ptr_type); + } + return compute_function_params; +} + llvm::Argument* IrEmitter::GetResultArgument() { return GetArg(compute_function_, 0); } llvm::Argument* IrEmitter::GetProfileCountersArgument() { - const int64 arg_index = IsParallelContext() ? 5 : 4; + const int64 arg_index = num_dynamic_loop_bounds_ > 0 ? 5 : 4; return hlo_to_profile_idx_ ? GetArg(compute_function_, arg_index) : nullptr; } @@ -2845,18 +2861,11 @@ llvm::Value* IrEmitter::EmitElementFunctionCall( AsStringRef(tensorflow::strings::StrCat(name, "_return_value"))); } -// Emits a core function call based on the following pseudo-code. -// -// char** parameter_addresses_buffer = -// allocate buffer with a pointer for each parameter to the function -// for each parameter index, i.e. for i = 0, ..., #parameters: -// parameter_addresses_buffer[i] = parameter_addresses[i] -// call function(return_value_buffer, -// parameter_addresses_buffer, -// temps) -// return return_value_buffer -- address of the return value. -void IrEmitter::EmitArrayFunctionCallInto( - llvm::Function* function, +// Emits code to allocate an array of parameter address pointers, and store +// each address from 'parameter_addresses'. +// Returns an array of compute function call arguments (including parameter +// address buffer). +std::vector IrEmitter::GetArrayFunctionCallArguments( tensorflow::gtl::ArraySlice parameter_addresses, llvm::Value* return_value_buffer, tensorflow::StringPiece name) { llvm::Value* parameter_addresses_buffer = @@ -2885,7 +2894,26 @@ void IrEmitter::EmitArrayFunctionCallInto( if (auto* profile_counters = GetProfileCountersArgument()) { arguments.push_back(profile_counters); } - ir_builder_.CreateCall(function, arguments); + return arguments; +} + +// Emits a core function call based on the following pseudo-code. +// +// char** parameter_addresses_buffer = +// allocate buffer with a pointer for each parameter to the function +// for each parameter index, i.e. for i = 0, ..., #parameters: +// parameter_addresses_buffer[i] = parameter_addresses[i] +// call function(return_value_buffer, +// parameter_addresses_buffer, +// temps) +// return return_value_buffer -- address of the return value. +void IrEmitter::EmitArrayFunctionCallInto( + llvm::Function* function, + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* return_value_buffer, tensorflow::StringPiece name) { + ir_builder_.CreateCall( + function, GetArrayFunctionCallArguments(parameter_addresses, + return_value_buffer, name)); } llvm::Value* IrEmitter::EmitArrayFunctionCall( @@ -2905,6 +2933,110 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall( return return_value_buffer; } +// Emits a call to a runtime fork/join function which dispatches parallel +// calls to 'parallel_function' (and joins threads before returning). +Status IrEmitter::EmitParallelForkJoin( + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* output_address, HloComputation* computation, + llvm::Function* parallel_function) { + HloInstruction* root = computation->root_instruction(); + + // Build ParallelForkJoin function type. + std::vector compute_function_params = GetComputeFunctionParams(); + // Number of parallel compute functions. + compute_function_params.push_back(ir_builder_.getInt32Ty()); + // Array of partitions. There is an array element for each + // partition x partition_dim x 2 (for dimension start and limit). + compute_function_params.push_back( + llvm::Type::getInt64PtrTy(module_->getContext())); + // Number of partitioned most-major dimensions in 'root.shape'. + compute_function_params.push_back(ir_builder_.getInt32Ty()); + // Function pointer for compute function to be dispatched in parallel. + compute_function_params.push_back( + llvm::Type::getInt8PtrTy(module_->getContext())); + + llvm::FunctionType* fork_join_type = llvm::FunctionType::get( + /*Result=*/llvm::Type::getVoidTy(module_->getContext()), + /*Params=*/compute_function_params, + /*isVarArg=*/false); + + llvm::Function* fork_join_func = + llvm::cast(module_->getOrInsertFunction( + runtime::kParallelForkJoinSymbolName, fork_join_type)); + fork_join_func->setCallingConv(llvm::CallingConv::C); + fork_join_func->setDoesNotThrow(); + + // Add common compute function arguments. + const string name = computation->name(); + std::vector arguments = + GetArrayFunctionCallArguments(parameter_addresses, output_address, name); + + // Create ShapePartitionIterator to generate all partitions of 'root.shape'. + ShapePartitionIterator partition_iterator(root->shape(), + root->outer_dimension_partitions()); + const int64 num_partitions = partition_iterator.GetTotalPartitionCount(); + // Add argument specifying the number of parallel partitions. + arguments.push_back(ir_builder_.getInt32(num_partitions)); + + // The number of partitioned most-major dimensions in 'root.shape'. + const int32 num_partitioned_dims = root->outer_dimension_partitions().size(); + // A dimension partition consists of two elements: [start_index, limit_index). + const int32 dim_partition_size = 2; + // Calculate array partition stride. + const int32 array_partition_stride = + num_partitioned_dims * dim_partition_size; + // Calculate the total number of elements in the partition array. + const int32 partition_array_size = + dim_partition_size * num_partitioned_dims * num_partitions; + + // Store dimension partition values as llvm constants in 'partitions'. + // See comments in runtime_fork_join.cc for array layout description. + std::vector partitions(partition_array_size); + for (int32 i = 0; i < num_partitions; ++i) { + std::vector> dim_partitions = + partition_iterator.GetPartition(i); + CHECK_EQ(num_partitioned_dims, dim_partitions.size()); + const int32 partition_index = i * array_partition_stride; + for (int32 j = 0; j < num_partitioned_dims; ++j) { + const std::pair& dim_partition = dim_partitions[j]; + const int32 index = partition_index + j * dim_partition_size; + // Store partition [dim_start, dim_limit) intervals for each dimension. + partitions[index] = ir_builder_.getInt64(dim_partition.first); + partitions[index + 1] = + ir_builder_.getInt64(dim_partition.first + dim_partition.second); + } + } + + // Create global variable out of dimension partitions in 'partitions'. + llvm::ArrayType* partitions_array_type = + llvm::ArrayType::get(ir_builder_.getInt64Ty(), partition_array_size); + llvm::Constant* partitions_array = + llvm::ConstantArray::get(partitions_array_type, partitions); + llvm::GlobalVariable* global_partitions_array = new llvm::GlobalVariable( + /*Module=*/*module_, + /*Type=*/partitions_array_type, + /*isConstant=*/true, + /*Linkage=*/llvm::GlobalValue::PrivateLinkage, + /*Initializer=*/partitions_array, + /*Name=*/ + AsStringRef( + tensorflow::strings::StrCat(name, "_parallel_dimension_partitions"))); + + // Add argument specifying parallel dimension partitions. + arguments.push_back(ir_builder_.CreateBitCast( + global_partitions_array, + llvm::Type::getInt64PtrTy(module_->getContext()))); + // Add argument specifying the number of partitioned most-major dimensions. + arguments.push_back(ir_builder_.getInt32(num_partitioned_dims)); + // Add argument for parallel compute function pointer. + arguments.push_back( + ir_builder_.CreateBitCast(parallel_function, ir_builder_.getInt8PtrTy())); + // Emit call to parallel fork/join. + ir_builder_.CreateCall(fork_join_func, arguments); + + return Status::OK(); +} + Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) { llvm::Value* addr; const Shape& target_shape = op->shape(); diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 53c4b6f241..58c185af1e 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -249,6 +249,9 @@ class IrEmitter : public DfsHloVisitorWithDefault { // Convenience function to get the IR type matching the given shape. llvm::Type* IrShapeType(const Shape& shape); + // Returns an array of compute function parameter types. + std::vector GetComputeFunctionParams(); + // Get the llvm::Value* that represents the "retval" argument of the // computation function being emitted by this emitter. llvm::Argument* GetResultArgument(); @@ -323,6 +326,18 @@ class IrEmitter : public DfsHloVisitorWithDefault { tensorflow::gtl::ArraySlice parameter_addresses, tensorflow::StringPiece name); + // Returns an array of compute function call arguments. + std::vector GetArrayFunctionCallArguments( + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* return_value_buffer, tensorflow::StringPiece name); + + // Emits a call to a runtime fork/join function which dispatches parallel + // calls to 'parallel_function' (and joins threads before returning). + Status EmitParallelForkJoin( + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* output_address, HloComputation* computation, + llvm::Function* parallel_function); + // Verifies that the element types of all of the given operand instructions // match and are of one of the given supported types. Status ElementTypesSameAndSupported( @@ -596,12 +611,6 @@ class IrEmitter : public DfsHloVisitorWithDefault { Status EmitXfeedTransfer(XfeedKind kind, const Shape& shape, llvm::Value* program_buffer_address); - // Returns true if the current function being emitted is called in a - // parallel context (returns false otherwise). - bool IsParallelContext() { - return parallel_cpu_backend_ && is_top_level_computation_; - } - const HloModuleConfig& hlo_module_config_; const bool parallel_cpu_backend_; diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc index d4b5e41f50..5afb2e67ff 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc +++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc @@ -48,29 +48,56 @@ class SimpleCostModel : public ParallelCostModel { class DefaultCostModel : public ParallelCostModel { public: DefaultCostModel(const int64 max_parallelism, + const HloCostAnalysis::ShapeSizeFunction& shape_size, std::unique_ptr cost_analysis) : max_parallelism_(max_parallelism), + shape_size_(shape_size), cost_analysis_(std::move(cost_analysis)) {} ~DefaultCostModel() override {} int64 GetParallelTaskCount(HloInstruction* instruction) override { - // Calculate the instruction cost in cycles. - // TODO(29630486) Improve on this linear cost model. - // Consider making 'min_cost_per_thread' be a function of the target - // bandwidth limit for instructions with low arithmetic complexity. - const int64 instruction_cost = - 1 * cost_analysis_->flop_count(*instruction) + - 2 * cost_analysis_->transcendental_count(*instruction) + - 10 * cost_analysis_->bytes_accessed(*instruction); - // Minimum per-thread cost is 100us of work on a 2GHz core. - const int64 min_cost_per_thread = 100000; + // Parameters for parallel task count computation. + int64 instruction_cost; + int64 min_cost_per_thread; + int64 max_parallelism; + // Calculate flops-to-bytes-ratio for 'instruction'. + const int64 bytes_accessed = + std::max(1LL, cost_analysis_->bytes_accessed(*instruction)); + const float flops_to_bytes_ratio = + cost_analysis_->flop_count(*instruction) / + static_cast(bytes_accessed); + // Check for I/O bound instructions. + if (flops_to_bytes_ratio <= 1.0) { + // Limit max parallelism for I/O bound instructions by assuming a + // sub-linear scaling function (fit based on empirical benchmark results). + // TODO(29630486) Develop system bandwidth model. + max_parallelism = + std::ceil(std::sqrt(tensorflow::port::NumSchedulableCPUs())); + // Use shape size instruction cost and L2 cache size min per-thread cost. + instruction_cost = shape_size_(instruction->shape()); + min_cost_per_thread = 256LL << 10; // 256KB L2 Cache size. + } else { + // Use max parallelism for compute bound instructions. + max_parallelism = max_parallelism_; + // Calculate the instruction cost in cycles. + // TODO(29630486) Improve on this linear cost model. + // Consider making 'min_cost_per_thread' be a function of the target + // bandwidth limit for instructions with low arithmetic complexity. + instruction_cost = + 1 * cost_analysis_->flop_count(*instruction) + + 2 * cost_analysis_->transcendental_count(*instruction) + + 10 * cost_analysis_->bytes_accessed(*instruction); + // Minimum per-thread cost is 100us of work on a 2GHz core. + min_cost_per_thread = 100000; + } // Return target parallel task count in [1, max_parallelism_]. - return std::min(max_parallelism_, + return std::min(max_parallelism, std::max(1LL, instruction_cost / min_cost_per_thread)); } private: const int64 max_parallelism_; + const HloCostAnalysis::ShapeSizeFunction shape_size_; const std::unique_ptr cost_analysis_; }; @@ -86,7 +113,7 @@ ParallelTaskAssignment::ParallelTaskAssignment( Status status = computation->root_instruction()->Accept(cost_analysis.get()); if (status.ok()) { // Set default cost model based on 'cost_analysis'. - cost_model_.reset(new DefaultCostModel(max_parallelism, + cost_model_.reset(new DefaultCostModel(max_parallelism, shape_size, std::move(cost_analysis))); } else { // Fall back to a simple cost model based on hlo size and L2 cache size. @@ -121,5 +148,102 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount( return cost_model_->GetParallelTaskCount(instruction); } +StatusOr ParallelTaskAssigner::Run(HloModule* module) { + XLA_VLOG_LINES(2, "ParallelTaskAssigner ENTRY"); + XLA_VLOG_LINES(3, module->ToString()); + + // Compute target parallel task counts for all instructions in 'module'. + HloToParallelTasks hlo_to_parallel_tasks; + ComputeTargetParallelTasks(module, &hlo_to_parallel_tasks); + + // Assign parallel tasks to target specific instructions in 'module'. + // TODO(b/27458679) Support inter-op parallelism. + bool changed = AssignParallelTasks(module, hlo_to_parallel_tasks); + + XLA_VLOG_LINES(2, "ParallelTaskAssigner EXIT"); + XLA_VLOG_LINES(3, module->ToString()); + return changed; +} + +bool ParallelTaskAssigner::AssignParallelTasks( + HloModule* module, const HloToParallelTasks& hlo_to_parallel_tasks) { + return AssignParallelTasksHelper(module, module->entry_computation(), + hlo_to_parallel_tasks); +} + +bool ParallelTaskAssigner::AssignParallelTasksHelper( + HloModule* module, HloComputation* computation, + const HloToParallelTasks& hlo_to_parallel_tasks) { + bool changed = false; + // Snapshot set of instructions because outlining modifies the set below. + std::vector instructions(computation->instructions().begin(), + computation->instructions().end()); + for (auto* instruction : instructions) { + // Assign parallel tasks to sub-computations for While and Call HLOs. + // TODO(b/27458679) Evaluate alternative intra-op parallelsim placement, + // and support other callable computations like reduce. + if (instruction->opcode() == HloOpcode::kWhile) { + changed |= AssignParallelTasksHelper(module, instruction->while_body(), + hlo_to_parallel_tasks); + continue; + } else if (instruction->opcode() == HloOpcode::kCall) { + changed |= AssignParallelTasksHelper(module, instruction->to_apply(), + hlo_to_parallel_tasks); + continue; + } + // Skip if no parallel tasks were computed in first pass. + auto it = hlo_to_parallel_tasks.find(instruction); + if (it == hlo_to_parallel_tasks.end()) { + continue; + } + // Get target parallel task count computed for 'instruction'. + const int64 target_parallel_task_count = (*it).second; + // Assign feasible dimension partitions (based on actual dimension sizes). + auto dim_partition_counts = ShapePartitionAssigner(instruction->shape()) + .Run(target_parallel_task_count); + const int64 total_partition_count = + ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts); + if (total_partition_count <= 1) { + // Feasible partition calculation resulting in no partitioning, so skip. + continue; + } + + // Outline 'instruction' in 'computation' for parallel task assignment. + auto* call = module->OutlineExpressionFromComputation( + {instruction}, + tensorflow::strings::StrCat("parallel_", instruction->name()), + computation); + + // Set assigned dimension partitioning to 'instruction'. + auto* new_root = call->to_apply()->root_instruction(); + new_root->set_outer_dimension_partitions(dim_partition_counts); + + VLOG(2) << "Assigned parallel task count: " << total_partition_count + << " to instruction: " << new_root->name() + << " parent: " << new_root->parent()->name(); + changed = true; + } + return changed; +} + +void ParallelTaskAssigner::ComputeTargetParallelTasks( + HloModule* module, HloToParallelTasks* hlo_to_parallel_tasks) { + // Compute parallel task counts for all instructions in 'module'. + for (auto* computation : module->computations()) { + if (computation->IsFusionComputation()) { + continue; + } + for (auto* instruction : computation->instructions()) { + // Query ParallelTaskAssignment for target parallel task count. + const int64 target_parallel_task_count = + parallel_task_assignment_.GetTargetParallelTaskCount(instruction); + if (target_parallel_task_count > 1) { + hlo_to_parallel_tasks->insert( + {instruction, target_parallel_task_count}); + } + } + } +} + } // namespace cpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h index 15f065a3ad..e036da5784 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h +++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h" #include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" namespace xla { namespace cpu { @@ -49,6 +50,54 @@ class ParallelTaskAssignment { std::unique_ptr cost_model_; }; +// ParallelTaskAssigner computes target parallel task counts for all HLOs +// in the module, then assigns parallel task counts to HLOs in the entry +// computation, or to HLOs in embedded computations invoked by (potentially +// nested) kWhile or kCall instructions. +// Each HLO which is assigned parallel task counts is outlined into its +// own embedded computation, which is compiled as a parallel compute function, +// and which is invoked from a kCall instruction that is lowered in codegen to +// a runtime parallel fork/join call. +class ParallelTaskAssigner : public HloPassInterface { + public: + // 'max_parallelism': the maximum parallel task count per instruction. + // 'shape_size': shape size function used by HloCostAnalysis during parallel + // task assignment. + // 'module': the containing HloModule. + ParallelTaskAssigner(const int64 max_parallelism, + const HloCostAnalysis::ShapeSizeFunction& shape_size, + HloModule* module) + : parallel_task_assignment_(max_parallelism, shape_size, module) {} + ~ParallelTaskAssigner() override {} + + tensorflow::StringPiece name() const override { + return "cpu-parallel-task-assigner"; + } + + // Run parallel task assigner on 'module'. + // Returns true if the computation was changed, false otherwise. + StatusOr Run(HloModule* module) override; + + private: + using HloToParallelTasks = std::unordered_map; + + // Assigns target parallel tasks from 'hlo_to_parallel_tasks' to HLOs in + // 'module'. + // Returns true if the computation was changed, false otherwise. + bool AssignParallelTasks(HloModule* module, + const HloToParallelTasks& hlo_to_parallel_tasks); + bool AssignParallelTasksHelper( + HloModule* module, HloComputation* computation, + const HloToParallelTasks& hlo_to_parallel_tasks); + + // Computes target parallel task counts (returned in 'parallel_task_counts') + // for parallelizable instructions in 'module'. + void ComputeTargetParallelTasks(HloModule* module, + HloToParallelTasks* hlo_to_parallel_tasks); + + ParallelTaskAssignment parallel_task_assignment_; +}; + } // namespace cpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc new file mode 100644 index 0000000000..d03da46575 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc @@ -0,0 +1,97 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h" + +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/compiler/xla/executable_run_options.h" +#include "tensorflow/core/lib/core/blocking_counter.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +using tensorflow::int32; +using tensorflow::int64; +using tensorflow::uint64; + +using ComputeFunctionType = void (*)(void*, const void*, const void**, void**, + int64*, uint64*); + +// Dispatches 'num_partitions - 1' calls to 'function_ptr' in parallel. +// Calls 'function_ptr' for first partition inline. +// Uses blocking counter to synchonize threads after parallel calls complete. +// +// The 'partitions' array has a total number of elements equal to +// 'num_partitions * num_partitioned_dims * 2' (the '2' is necessary to specify +// dimension start and limit indices). +// +// The 'partitions' array layout stores array elements in memory with dimension +// start limit as the most-minor dimension, followed by dimension, then +// partition. +// +// EX: Layout of 'partitions' array with 'num_partitions = 2', and +// 'num_partitioned_dims = 3' +// +// [partition0_dim0_start] +// [partition0_dim0_limit] +// [partition0_dim1_start] +// [partition0_dim1_limit] +// [partition0_dim2_start] +// [partition0_dim2_limit] +// [partition1_dim0_start] +// [partition1_dim0_limit] +// [partition1_dim1_start] +// [partition1_dim1_limit] +// [partition1_dim2_start] +// [partition1_dim2_limit] +// +void __xla_cpu_runtime_ParallelForkJoin( + void* result_ptr, const void* run_options_ptr, const void** params, + void** temps, uint64* prof_counters, int32 num_partitions, + int64* partitions, int32 num_partitioned_dims, void* function_ptr) { + VLOG(2) << "ParallelForkJoin ENTRY" + << " num_partitions: " << num_partitions + << " num_partitioned_dims: " << num_partitioned_dims; + CHECK_GT(num_partitions, 1); + CHECK_GT(num_partitioned_dims, 0); + const xla::ExecutableRunOptions* run_options = + static_cast(run_options_ptr); + ComputeFunctionType function = + reinterpret_cast(function_ptr); + // Compute partition stride in 'partitions' array. + const int64 stride = 2 * num_partitioned_dims; + + // Dispatch 'num_partitions - 1' compute functions to run in parallel. + tensorflow::BlockingCounter bc(num_partitions - 1); + for (int32 i = 1; i < num_partitions; ++i) { + const int64 offset = i * stride; + run_options->intra_op_thread_pool()->enqueueNoNotification( + [i, function, result_ptr, run_options_ptr, params, temps, prof_counters, + partitions, offset, &bc]() { + function(result_ptr, run_options_ptr, params, temps, + &partitions[offset], prof_counters); + bc.DecrementCount(); + VLOG(3) << "ParallelForkJoin partition " << i << " done."; + }); + } + + // Call first compute function inline. + function(result_ptr, run_options_ptr, params, temps, &partitions[0], + prof_counters); + VLOG(3) << "ParallelForkJoin partition 0 done."; + bc.Wait(); + VLOG(2) << "ParallelForkJoin EXIT"; +} diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h new file mode 100644 index 0000000000..fcf1cc6207 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h @@ -0,0 +1,33 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_ +#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_ + +#include "tensorflow/core/platform/types.h" + +extern "C" { + +// Dispatches 'num_partitions' parallel calls to 'function_ptr' and joins +// threads before returning. See comments in runtime_fork_join.cc for details. +extern void __xla_cpu_runtime_ParallelForkJoin( + void* result_ptr, const void* run_options_ptr, const void** params, + void** temps, tensorflow::uint64* prof_counters, + tensorflow::int32 num_partitions, tensorflow::int64* partitions, + tensorflow::int32 num_partitioned_dims, void* function_ptr); + +} // extern "C" + +#endif // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_ diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc index c614e334a8..cfffb3fbc3 100644 --- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc +++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h" #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h" #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h" +#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h" #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h" #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h" #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h" @@ -104,6 +105,7 @@ class JITSymbolTable { ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32); ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32); ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64); + ADD_JIT_SYMBOL_TO_TABLE(ParallelForkJoin); #undef ADD_JIT_SYMBOL_TO_TABLE } diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 769f509adc..b02d906d93 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -1411,8 +1411,10 @@ xla_test( "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:core_cpu_internal", "//tensorflow/core:lib", "//tensorflow/core:test", + "//third_party/eigen3", ], ) diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc index 3bf9ccb197..a8f6488996 100644 --- a/tensorflow/compiler/xla/tests/fusion_test.cc +++ b/tensorflow/compiler/xla/tests/fusion_test.cc @@ -17,8 +17,12 @@ limitations under the License. #include #include #include +#include #include +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/client/client_library.h" #include "tensorflow/compiler/xla/client/computation.h" @@ -37,6 +41,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" @@ -250,6 +255,42 @@ XLA_TEST_F(FusionTest, Parameter) { ErrorSpec(1e-4)); } +XLA_TEST_F(FusionTest, RandomizedParallelPartition) { + // Tests parallel partitioning of a fusion instruction. + // Create shape with random outer dimension size to generate random parallel + // partition counts for each test run. + const int seed = tensorflow::testing::RandomSeed(); + LOG(INFO) << "RandomizedParallelPartition seed: " << seed; + std::mt19937 generator(seed); + std::uniform_int_distribution distribution(128, 1024); + const int64 rand_dim0_size = distribution(generator); + const int64 dim1_size = 1024; + Shape shape = + ShapeUtil::MakeShapeWithLayout(F32, {rand_dim0_size, dim1_size}, {1, 0}); + // Build simple fusion computation: y = x^2 (elementwise). + auto builder = HloComputation::Builder(TestName()); + auto hlo_module = CreateNewModule(); + + auto two = builder.AddInstruction( + HloInstruction::CreateConstant(Literal::CreateR0(2.0))); + auto x = + builder.AddInstruction(HloInstruction::CreateBroadcast(shape, two, {})); + auto y = builder.AddInstruction( + HloInstruction::CreateBinary(shape, HloOpcode::kMultiply, x, x)); + + hlo_module->AddEntryComputation(builder.Build()) + ->CreateFusionInstruction(/*instructions_to_fuse=*/{y, x, two}, + HloInstruction::FusionKind::kLoop); + // Compute result. + auto result = ExecuteAndTransfer(std::move(hlo_module), {}); + // Every element of result should be y = x^2 = 4.0. + for (int i = 0; i < rand_dim0_size; ++i) { + for (int j = 0; j < dim1_size; ++j) { + EXPECT_EQ(4.0, result->Get({i, j})); + } + } +} + XLA_TEST_F(FusionTest, BroadcastIntoBinaryOp) { auto builder = HloComputation::Builder(TestName()); auto hlo_module = CreateNewModule(); @@ -722,47 +763,104 @@ void BM_ParallelFusion(int num_iters) { auto executors = PlatformUtil::GetStreamExecutors(platform).ValueOrDie(); StreamExecutorMemoryAllocator allocator(platform, executors); - const int64 intra_op_parallelism_threads = 16; + const int64 intra_op_parallelism_threads = 24; xla::LocalClientOptions client_options; client_options.set_platform(platform); client_options.set_intra_op_parallelism_threads(intra_op_parallelism_threads); auto client = ClientLibrary::GetOrCreateLocalClient(client_options).ValueOrDie(); - const int64 dim_size = 1024; - // Create a simple fusable elementwise computation. + auto* transfer_manager = + TransferManager::GetForPlatform(platform).ValueOrDie(); + int device_ordinal = client->default_device_ordinal(); + + // Computation shape parameters. + const int64 param0_dim0 = 1024; + const int64 param0_dim1 = 1024; + const int64 param1_dim0 = 1024; + const int64 param1_dim1 = 1024; + const int64 param2_dim0 = 1024; + const int64 param2_dim1 = 1024; + + // Create computation. ComputationBuilder builder(client, "ParallelFusion"); - Shape input_shape = ShapeUtil::MakeShape(F32, {dim_size, dim_size}); - auto input0 = builder.Broadcast(builder.ConstantR0(1.5f), - AsInt64Slice(input_shape.dimensions())); - auto input1 = builder.Broadcast(builder.ConstantR0(2.0f), - AsInt64Slice(input_shape.dimensions())); - auto input2 = builder.Broadcast(builder.ConstantR0(3.0f), - AsInt64Slice(input_shape.dimensions())); - auto x = builder.Mul(input0, input1); - auto y = builder.Add(x, input2); + Shape shape0 = ShapeUtil::MakeShape(F32, {param0_dim0, param0_dim1}); + auto param0 = builder.Parameter(0, shape0, "param0"); + Shape shape1 = ShapeUtil::MakeShape(F32, {param1_dim0, param1_dim1}); + auto param1 = builder.Parameter(1, shape1, "param1"); + Shape shape2 = ShapeUtil::MakeShape(F32, {param2_dim0, param2_dim1}); + auto param2 = builder.Parameter(2, shape2, "param2"); + + auto x = builder.Mul(param0, param1); + auto y = builder.Add(x, param2); auto computation = builder.Build().ConsumeValueOrDie(); + // Transfer literals to device. + auto buffer0 = + ScopedShapedBuffer::Allocate(shape0, &allocator, /*device_ordinal=*/0) + .ConsumeValueOrDie(); + auto param0_literal = + Literal::CreateR2F32Linspace(1.0, 2.0, param0_dim0, param0_dim1); + ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( + executors[device_ordinal], *param0_literal, buffer0->mutable_buffer({}))); + + auto buffer1 = + ScopedShapedBuffer::Allocate(shape1, &allocator, /*device_ordinal=*/0) + .ConsumeValueOrDie(); + auto param1_literal = + Literal::CreateR2F32Linspace(1.0, 2.0, param1_dim0, param1_dim1); + ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( + executors[device_ordinal], *param1_literal, buffer1->mutable_buffer({}))); + + auto buffer2 = + ScopedShapedBuffer::Allocate(shape2, &allocator, /*device_ordinal=*/0) + .ConsumeValueOrDie(); + auto param2_literal = + Literal::CreateR2F32Linspace(1.0, 2.0, param2_dim0, param2_dim1); + ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( + executors[device_ordinal], *param2_literal, buffer2->mutable_buffer({}))); + + // Build executable. std::unique_ptr executable = - client->Compile(computation, {}, ExecutableBuildOptions()) + client + ->Compile(computation, + {&buffer0->shape(), &buffer1->shape(), &buffer2->shape()}, + ExecutableBuildOptions()) .ConsumeValueOrDie(); - // Run some warm-up executions. + se::Stream stream(executors[client->default_device_ordinal()]); + stream.Init(); + + // Initialize thread pool. + tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "XLAEigen", + intra_op_parallelism_threads); + tensorflow::EigenThreadPoolWrapper tp(&pool); + Eigen::ThreadPoolDevice device(&tp, tp.NumThreads()); + + // Initialize ExecutableRunOptions. ExecutableRunOptions options; - options.set_allocator(&allocator); + options.set_allocator(&allocator).set_stream(&stream); + options.set_intra_op_thread_pool(&device); + + // Run some warm-up executions. const int kWarmups = 2; for (int i = 0; i < kWarmups; ++i) { - auto result = executable->Run({}, options); + auto result = + executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options); ASSERT_TRUE(result.ok()); } // Run benchmark. - tensorflow::testing::BytesProcessed(static_cast(num_iters) * dim_size * - dim_size * sizeof(float)); + const int64 total_bytes = param0_dim0 * param0_dim0 + + param1_dim0 * param1_dim0 + + param2_dim0 * param2_dim0; + tensorflow::testing::BytesProcessed(static_cast(num_iters) * + total_bytes * sizeof(float)); tensorflow::testing::UseRealTime(); tensorflow::testing::StartTiming(); for (int i = 0; i < num_iters; ++i) { - auto result = executable->Run({}, options); + auto result = + executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options); ASSERT_TRUE(result.ok()); } } -- GitLab From 05aebd4c342c3ab432250fa3ef17bf212061f931 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Mon, 16 Oct 2017 08:11:06 -0700 Subject: [PATCH 009/573] tfdbg doc: Fix minor typo PiperOrigin-RevId: 172326303 --- tensorflow/docs_src/programmers_guide/debugger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md index 3ede42e8f7..58154d19e7 100644 --- a/tensorflow/docs_src/programmers_guide/debugger.md +++ b/tensorflow/docs_src/programmers_guide/debugger.md @@ -141,7 +141,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at | **`lt`** | | **List dumped tensors.** | `lt` | | | `-n ` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` | | | `-t ` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` | -| | `s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | +| | `-s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | | | `-r` | Sort in reverse order. | `lt -r -s dump_size` | | **`pt`** | | **Print value of a dumped tensor.** | | | | `pt ` | Print tensor value. | `pt hidden/Relu:0` | -- GitLab From 51c115b33fecd9e96aa12c264c2c717afe8bfea8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 09:03:28 -0700 Subject: [PATCH 010/573] Fix typo (undefined variable `mean_absolute_error`, should refer to `error` previously defined). PiperOrigin-RevId: 172331504 --- tensorflow/docs_src/api_guides/python/contrib.metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/api_guides/python/contrib.metrics.md b/tensorflow/docs_src/api_guides/python/contrib.metrics.md index b502826e6a..1eb9cf417a 100644 --- a/tensorflow/docs_src/api_guides/python/contrib.metrics.md +++ b/tensorflow/docs_src/api_guides/python/contrib.metrics.md @@ -64,7 +64,7 @@ sess.run(tf.local_variables_initializer()) for batch in range(num_batches): sess.run([update_op_acc, update_op_error]) -accuracy, mean_absolute_error = sess.run([accuracy, mean_absolute_error]) +accuracy, error = sess.run([accuracy, error]) ``` Note that when evaluating the same metric multiple times on different inputs, -- GitLab From 14a66fd59c75f1b75c3e32a7e243778d3e83d221 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 09:21:09 -0700 Subject: [PATCH 011/573] [TF:XLA] Update xla_data comments for And, Or, and Not. PiperOrigin-RevId: 172333451 --- tensorflow/compiler/xla/xla_data.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 0d7e583bed..eae284afb7 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -621,7 +621,7 @@ message WhileRequest { enum UnaryOperation { UNOP_INVALID = 0; - // Elementwise, logical negation + // Elementwise, logical negation on booleans and bitwise negation on ints. UNOP_NOT = 1; // Elementwise, computes e^x. @@ -710,7 +710,7 @@ enum BinaryOperation { // Remainder operation. BINOP_REM = 17; - // Logical operators + // Element-wise, logical operators on booleans and bitwise operators on ints. BINOP_AND = 18; BINOP_OR = 19; -- GitLab From 0a572887ffa9879d6a303aafd6de9e288776fc8f Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Mon, 16 Oct 2017 09:43:42 -0700 Subject: [PATCH 012/573] Automated g4 rollback of changelist 172039259 PiperOrigin-RevId: 172336111 --- tensorflow/python/BUILD | 1 - .../resource_variable_ops_test.py | 10 -- .../python/ops/resource_variable_ops.py | 5 - tensorflow/python/training/adam_test.py | 82 ++++++++-------- tensorflow/python/training/saver_test.py | 97 +++++++++---------- 5 files changed, 88 insertions(+), 107 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 1885caf695..48436fe8cf 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3396,7 +3396,6 @@ cuda_py_test( ":training", ":platform_test", ":client_testlib", - ":variable_scope", "//third_party/py/numpy", ], ) diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 6f2bc2f752..8cf8286ed1 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -422,16 +422,6 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(1, v1.read_value().numpy()) self.assertEqual(2, v2.read_value().numpy()) - def testDestruction(self): - with context.eager_mode(): - var = resource_variable_ops.ResourceVariable(initial_value=1.0, - name="var8") - var.__del__() - with self.assertRaisesRegexp(errors.NotFoundError, - r"Resource .*\/var8\/.* does not exist."): - resource_variable_ops.destroy_resource_op(var._handle, - ignore_lookup_error=False) - if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 99ff02873b..cbfa141256 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -427,11 +427,6 @@ class ResourceVariable(variables.Variable): self._constraint = None # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py) - def __del__(self): - if context.in_eager_mode(): - gen_resource_variable_ops.destroy_resource_op(self._handle, - ignore_lookup_error=False) - @property def dtype(self): """The dtype of this variable.""" diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py index 96de9b921b..defcf33714 100644 --- a/tensorflow/python/training/adam_test.py +++ b/tensorflow/python/training/adam_test.py @@ -29,7 +29,6 @@ from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import adam @@ -153,54 +152,53 @@ class AdamOptimizerTest(test.TestCase): def doTestBasic(self, use_resource=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with variable_scope.variable_scope("%d" % i): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) + + if use_resource: + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) - opt = adam.AdamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + opt = adam.AdamOptimizer() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - if context.in_graph_mode(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + if context.in_graph_mode(): + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - beta1_power, beta2_power = opt._get_beta_accumulators() + beta1_power, beta2_power = opt._get_beta_accumulators() - # Run 3 steps of Adam - for t in range(1, 4): - if context.in_graph_mode(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + # Run 3 steps of Adam + for t in range(1, 4): + if context.in_graph_mode(): + self.evaluate(update) + elif t > 1: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**(t + 1), - self.evaluate(beta2_power)) + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta2_power)) - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testBasic(self): with self.test_session(): diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index a8eb8e5fcf..07cd67a4b9 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -110,32 +110,32 @@ class SaverTest(test.TestCase): # Start a second session. In that session the parameter nodes # have not been initialized either. with self.test_session(graph=ops_lib.Graph()) as sess: - v0_2 = variable_op(-1.0, name="v0") - v1_2 = variable_op(-1.0, name="v1") - v2_2 = saver_test_utils.CheckpointedOp(name="v2") + v0 = variable_op(-1.0, name="v0") + v1 = variable_op(-1.0, name="v1") + v2 = saver_test_utils.CheckpointedOp(name="v2") # Assert that the variables are not initialized. if context.in_graph_mode(): self.assertEqual( len(variables.report_uninitialized_variables().eval()), 2) - self.assertEqual(0, len(v2_2.keys().eval())) - self.assertEqual(0, len(v2_2.values().eval())) + self.assertEqual(0, len(v2.keys().eval())) + self.assertEqual(0, len(v2.values().eval())) # Restore the saved values in the parameter nodes. - save = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable}) + save = saver_module.Saver({"v0": v0, "v1": v1, "v2": v2.saveable}) save.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, self.evaluate(v0_2)) - self.assertEqual(20.0, self.evaluate(v1_2)) - self.assertEqual(b"k1", self.evaluate(v2_2.keys())) - self.assertEqual(30.0, self.evaluate(v2_2.values())) + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) + self.assertEqual(b"k1", self.evaluate(v2.keys())) + self.assertEqual(30.0, self.evaluate(v2.values())) # Build another graph with 2 nodes, initialized # differently, and a Restore node for them. with self.test_session(graph=ops_lib.Graph()) as sess: - v0_3 = variable_op(1000.0, name="v0") - v1_3 = variable_op(2000.0, name="v1") - v2_3 = saver_test_utils.CheckpointedOp(name="v2") - v2_init = v2_3.insert("k1000", 3000.0) + v0_2 = variable_op(1000.0, name="v0") + v1_2 = variable_op(2000.0, name="v1") + v2_2 = saver_test_utils.CheckpointedOp(name="v2") + v2_init = v2_2.insert("k1000", 3000.0) # Check that the parameter nodes have been initialized. if context.in_graph_mode(): @@ -143,19 +143,19 @@ class SaverTest(test.TestCase): self.evaluate(init_all_op) # TODO(xpan): Why _mutable_hash_table_v2 doesn't create empty # table as it claims in eager mode? - self.assertEqual(b"k1000", self.evaluate(v2_3.keys())) - self.assertEqual(3000.0, self.evaluate(v2_3.values())) - self.assertEqual(1000.0, self.evaluate(v0_3)) - self.assertEqual(2000.0, self.evaluate(v1_3)) + self.assertEqual(b"k1000", self.evaluate(v2_2.keys())) + self.assertEqual(3000.0, self.evaluate(v2_2.values())) + self.assertEqual(1000.0, self.evaluate(v0_2)) + self.assertEqual(2000.0, self.evaluate(v1_2)) # Restore the values saved earlier in the parameter nodes. - save2 = saver_module.Saver({"v0": v0_3, "v1": v1_3, "v2": v2_3.saveable}) + save2 = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable}) save2.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, self.evaluate(v0_3)) - self.assertEqual(20.0, self.evaluate(v1_3)) - self.assertEqual(b"k1", self.evaluate(v2_3.keys())) - self.assertEqual(30.0, self.evaluate(v2_3.values())) + self.assertEqual(10.0, self.evaluate(v0_2)) + self.assertEqual(20.0, self.evaluate(v1_2)) + self.assertEqual(b"k1", self.evaluate(v2_2.keys())) + self.assertEqual(30.0, self.evaluate(v2_2.values())) def testBasic(self): self.basicSaveRestore(variables.Variable) @@ -487,10 +487,10 @@ class SaverTest(test.TestCase): val = save.save(sess, save_path) self.assertEqual(save_path, val) with self.test_session() as sess: - var2 = resource_variable_ops.ResourceVariable(other_value, name=var_name) - save = saver_module.Saver({var_name: var2}) + var = resource_variable_ops.ResourceVariable(other_value, name=var_name) + save = saver_module.Saver({var_name: var}) save.restore(sess, save_path) - self.assertAllClose(var_value, self.evaluate(var2)) + self.assertAllClose(var_value, self.evaluate(var)) def testCacheRereadsFile(self): save_path = os.path.join(self.get_temp_dir(), "cache_rereads") @@ -618,29 +618,28 @@ class SaverTest(test.TestCase): global_step_int = 5 # Save and reload one Variable named "var0". self._SaveAndLoad("var0", 0.0, 1.0, save_path) - for i, use_tensor in enumerate([True, False]): - with variable_scope.variable_scope("%d" % i): - var = resource_variable_ops.ResourceVariable(1.0, name="var0") - save = saver_module.Saver( - { - var._shared_name: var - }, pad_step_number=pad_step_number) - if context.in_graph_mode(): - self.evaluate(var.initializer) - sess = ops_lib.get_default_session() - else: - sess = None - if use_tensor: - global_step = constant_op.constant(global_step_int) - val = save.save(sess, save_path, global_step=global_step) - else: - val = save.save(sess, save_path, global_step=global_step_int) - if pad_step_number: - expected_save_path = "%s-%s" % (save_path, - "{:08d}".format(global_step_int)) - else: - expected_save_path = "%s-%d" % (save_path, global_step_int) - self.assertEqual(expected_save_path, val) + for use_tensor in [True, False]: + var = resource_variable_ops.ResourceVariable(1.0, name="var0") + save = saver_module.Saver( + { + var._shared_name: var + }, pad_step_number=pad_step_number) + if context.in_graph_mode(): + self.evaluate(var.initializer) + sess = ops_lib.get_default_session() + else: + sess = None + if use_tensor: + global_step = constant_op.constant(global_step_int) + val = save.save(sess, save_path, global_step=global_step) + else: + val = save.save(sess, save_path, global_step=global_step_int) + if pad_step_number: + expected_save_path = "%s-%s" % (save_path, + "{:08d}".format(global_step_int)) + else: + expected_save_path = "%s-%d" % (save_path, global_step_int) + self.assertEqual(expected_save_path, val) def testSaveWithGlobalStepWithPadding(self): self.testSaveWithGlobalStep(pad_step_number=True) -- GitLab From 07c6faf039ff1a49a540f8caafd7a30692fcf14d Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Mon, 16 Oct 2017 09:53:11 -0700 Subject: [PATCH 013/573] Adds a host-memory GPU kernel for DestroyResourceOp. PiperOrigin-RevId: 172337312 --- tensorflow/core/kernels/resource_variable_ops.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index 3cca493972..90db0c2b7b 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -200,6 +200,9 @@ class DestroyResourceOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("DestroyResourceOp").Device(DEVICE_CPU), DestroyResourceOp); +REGISTER_KERNEL_BUILDER( + Name("DestroyResourceOp").Device(DEVICE_GPU).HostMemory("resource"), + DestroyResourceOp); template class AssignVariableOp : public OpKernel { -- GitLab From 19fd294eae4e8e22f6ab46b21cf41323750a1c69 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Mon, 16 Oct 2017 10:09:55 -0700 Subject: [PATCH 014/573] Support ClusterSpec propagation with XLA Devices Currently, you cannot use ClusterSpec propagation in conjunction with XLA devices, as the RenamedDevice wraps the underlying device and breaks the dynamic cast. PiperOrigin-RevId: 172339725 --- tensorflow/compiler/jit/xla_device.cc | 4 +++- tensorflow/core/common_runtime/renamed_device.h | 7 +++++++ tensorflow/core/framework/device_base.h | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index a2c91511ec..7ccea58f6e 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/device_factory.h" #include "tensorflow/core/common_runtime/dma_helper.h" #include "tensorflow/core/common_runtime/function.h" +#include "tensorflow/core/common_runtime/renamed_device.h" #include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/framework/device_base.h" #include "tensorflow/core/framework/function.h" @@ -161,7 +162,8 @@ const DeviceType& XlaDevice::Metadata::jit_device_type() const { /* static */ Status XlaDevice::GetMetadata(OpKernelContext* ctx, const Metadata** metadata) { - XlaDevice* xla_device = dynamic_cast(ctx->device()); + XlaDevice* xla_device = + dynamic_cast(ctx->device()->UnderlyingDevice()); if (xla_device == nullptr) { return errors::Internal( "Cannot get XLA metadata from non-XLA device \"", ctx->device()->name(), diff --git a/tensorflow/core/common_runtime/renamed_device.h b/tensorflow/core/common_runtime/renamed_device.h index 0158e18ced..22a70fbdfa 100644 --- a/tensorflow/core/common_runtime/renamed_device.h +++ b/tensorflow/core/common_runtime/renamed_device.h @@ -37,6 +37,13 @@ class RenamedDevice : public Device { return underlying_->RequiresRecordingAccessedTensors(); } + const DeviceBase* UnderlyingDevice() const override { + return underlying_->UnderlyingDevice(); + } + DeviceBase* UnderlyingDevice() override { + return underlying_->UnderlyingDevice(); + } + const CpuWorkerThreads* tensorflow_cpu_worker_threads() const override { return underlying_->tensorflow_cpu_worker_threads(); } diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h index 14a96c57b5..33bd5d250c 100644 --- a/tensorflow/core/framework/device_base.h +++ b/tensorflow/core/framework/device_base.h @@ -188,6 +188,9 @@ class DeviceBase { // by GPU devices to return a derived type. virtual PerOpGpuDevice* MakeGpuDevice() { return nullptr; } + virtual DeviceBase* UnderlyingDevice() { return this; } + virtual const DeviceBase* UnderlyingDevice() const { return this; } + // This is overridden by GPU devices to reinitialize the derived // type returned by MakeGpuDevice. virtual void ReinitializeGpuDevice(OpKernelContext* /*context*/, -- GitLab From 3b595a805bbcf4be24a2e01abe1b8031d82dc57b Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Mon, 16 Oct 2017 10:13:58 -0700 Subject: [PATCH 015/573] Support a configurable TPU job name PiperOrigin-RevId: 172340173 --- .../contrib/tpu/python/tpu/tpu_config.py | 16 +++++-- .../contrib/tpu/python/tpu/tpu_estimator.py | 45 ++++++++++++++++++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py index 0a3be8503a..79fd8b839b 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py @@ -27,7 +27,10 @@ from tensorflow.python.estimator import run_config as run_config_lib class TPUConfig( collections.namedtuple('TPUConfig', [ - 'iterations_per_loop', 'num_shards', 'per_host_input_for_training' + 'iterations_per_loop', + 'num_shards', + 'per_host_input_for_training', + 'tpu_job_name', ])): """TPU related configuration required by `TPUEstimator`. @@ -46,12 +49,17 @@ class TPUConfig( that this only works for single-host TPU training now (tracked in b/67051042). For multi-host, please use Per-Core, i.e., `False` for `per_host_input_for_training`. + tpu_job_name: The name of the TPU job. Typically, this name is auto-inferred + within TPUEstimator, however when using ClusterSpec propagation in more + esoteric cluster configurations, you may need to specify the job name as a + string. """ def __new__(cls, iterations_per_loop=2, num_shards=2, - per_host_input_for_training=True): + per_host_input_for_training=True, + tpu_job_name=None): # Check iterations_per_loop. util_lib.check_positive_integer(iterations_per_loop, @@ -59,12 +67,12 @@ class TPUConfig( # Check num_shards. util_lib.check_positive_integer(num_shards, 'TPUConfig num_shards') - return super(TPUConfig, cls).__new__( cls, iterations_per_loop=iterations_per_loop, num_shards=num_shards, - per_host_input_for_training=per_host_input_for_training) + per_host_input_for_training=per_host_input_for_training, + tpu_job_name=tpu_job_name) class RunConfig(run_config_lib.RunConfig): diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 43f9defd54..de6c8140c6 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -122,12 +122,55 @@ def _increase_eval_step_op(iterations_per_loop): use_locking=True) +_DEFAULT_JOB_NAME = 'tpu_worker' +_DEFAULT_COORDINATOR_JOB_NAME = 'coordinator' +_LOCAL_MASTERS = ('', 'local') + + def _tpu_job(run_config, mode): + """Returns the job name to use to place TPU computations on. + + Args: + run_config: The tpu_config.RunConfig used for this custom estimator. + mode: A model_fn_lib.ModeKeys value. + + Returns: + A string containing the job name, or None if no job should be specified. + + Raises: + ValueError: If the user needs to specify a tpu_job_name, because we are + unable to infer the job name automatically, or if the user-specified job + names are inappropriate. + """ + # If the user specifies the tpu_job_name, use that. + if run_config.tpu_config.tpu_job_name: + return run_config.tpu_config.tpu_job_name + # The tpu job is determined by the run_config. Right now, this method is # required as tpu_config is not part of the RunConfig. master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL else run_config.master) - return None if master in ['', 'local'] else 'tpu_worker' + if master in _LOCAL_MASTERS: + return None + + if (not run_config.session_config or + not run_config.session_config.cluster_def.job): + return _DEFAULT_JOB_NAME + cluster_def = run_config.session_config.cluster_def + job_names = set([job.name for job in cluster_def.job]) + if _DEFAULT_JOB_NAME in job_names: + # b/37868888 tracks allowing ClusterSpec propagation to reuse job names. + raise ValueError('Currently, tpu_worker is not an allowed job name.') + if len(job_names) == 1: + return cluster_def.job[0].name + if len(job_names) == 2: + if _DEFAULT_COORDINATOR_JOB_NAME in job_names: + job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME) + return job_names.pop() + # TODO(b/67716447): Include more sophisticated heuristics. + raise ValueError( + 'Could not infer TPU job name. Please specify a tpu_job_name as part of ' + 'your TPUConfig.') def _is_running_on_cpu(use_tpu, mode, eval_batch_size): -- GitLab From 2b8ddee235cf5b18a56e3434a72150daaf107169 Mon Sep 17 00:00:00 2001 From: Jayaram Bobba Date: Mon, 16 Oct 2017 10:31:29 -0700 Subject: [PATCH 016/573] Use char* for mkl allocator strings instead of char[] to workaround build issues (#13697) --- tensorflow/core/common_runtime/mkl_cpu_allocator.h | 4 ++-- tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h index 5951b3b6a1..53e80b1ee3 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h @@ -51,7 +51,7 @@ class MklCPUAllocator : public Allocator { // Constructor and other standard functions /// Environment variable that user can set to upper bound on memory allocation - static constexpr const char kMaxLimitStr[] = "TF_MKL_ALLOC_MAX_BYTES"; + static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES"; /// Default upper limit on allocator size - 64GB static const size_t kDefaultMaxLimit = 64LL << 30; @@ -146,7 +146,7 @@ class MklCPUAllocator : public Allocator { static const bool kAllowGrowth = true; /// Name - static constexpr const char kName[] = "mklcpu"; + static constexpr const char* kName = "mklcpu"; /// The alignment that we need for the allocations static const size_t kAlignment = 64; diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc index cfefaa92e4..a67411cd2e 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc @@ -23,8 +23,6 @@ limitations under the License. namespace tensorflow { -constexpr char MklCPUAllocator::kMaxLimitStr[]; - TEST(MKLBFCAllocatorTest, TestMaxLimit) { AllocatorStats stats; setenv(MklCPUAllocator::kMaxLimitStr, "1000", 1); -- GitLab From 2ebbce8e82d8d07bee6b8be14a3961ebdef977a0 Mon Sep 17 00:00:00 2001 From: Se-won Kim <30789814+wonsekim@users.noreply.github.com> Date: Tue, 17 Oct 2017 02:31:58 +0900 Subject: [PATCH 017/573] fixed type error (API r1.3 document, tf.truncatediv) (#13712) It was just type error from '-7 / 5 = 1' to '-7 / 5 = -1' --- tensorflow/core/ops/math_ops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 74af6f7f4a..7b971a9fd5 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -623,7 +623,7 @@ REGISTER_OP("TruncateDiv") Returns x / y element-wise for integer types. Truncation designates that negative numbers will round fractional quantities -toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different +toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different than Python semantics. See `FloorDiv` for a division function that matches Python Semantics. -- GitLab From 5f62ef255eecc1a1e28a9ad91de63ea29cd97ef5 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 16 Oct 2017 10:32:58 -0700 Subject: [PATCH 018/573] Proper use of convert_to_tensor in custom_gradient PiperOrigin-RevId: 172342933 --- tensorflow/python/eager/backprop_test.py | 25 ++++++++++++++++++++++ tensorflow/python/eager/custom_gradient.py | 11 +--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 2409a7b198..d53c69afcc 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -475,6 +475,31 @@ class BackpropTest(test.TestCase): self.assertEqual(7, grad.numpy()) self.assertEqual(x, var) + def testCustomGradient(self): + + @custom_gradient.custom_gradient + def my_mul(x, y): + result = x*y + + def grad(dr): + return [dr*y, dr*x] + return result, grad + + lr = 0.25 + x = resource_variable_ops.ResourceVariable(2., name='x') + + def loss(x): + return my_mul(2., x.read_value()) + + loss_grads_fn = backprop.implicit_val_and_grad(loss) + + losses = [] + for _ in range(5): + loss, grads_and_vars = loss_grads_fn(x) + losses.append(loss.numpy()) + for (grad, var) in grads_and_vars: + var.assign_sub(lr*grad) + self.assertAllEqual(losses, [4.0, 3., 2., 1., 0.]) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py index df116dd819..4ac30075b2 100644 --- a/tensorflow/python/eager/custom_gradient.py +++ b/tensorflow/python/eager/custom_gradient.py @@ -22,7 +22,6 @@ from tensorflow.python.eager import context from tensorflow.python.eager import tape from tensorflow.python.framework import ops as tf_ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resource_variable_ops from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -69,19 +68,11 @@ def custom_gradient(f): return nest.pack_sequence_as( structure=result, flat_sequence=all_tensors[:len(flat_result)]) - input_tensors = [] - for x in args: - if isinstance(x, tf_ops.Tensor): - input_tensors.append(x) - if isinstance(x, resource_variable_ops.ResourceVariable): - input_tensors.append(x.read_value()) + input_tensors = [tf_ops.convert_to_tensor(x) for x in args] with tape.stop_recording(): result, grad_fn = f(*args, **kwargs) - # TODO(apassos): naive uses of custom_gradient will not get the correct - # second derivative this way if they capture any output tensors. Change the - # signature of custom_gradient. def actual_grad_fn(*outputs): return nest.flatten(grad_fn(*outputs)) -- GitLab From 2f916fcf2f49f8a30d8460c1c8d7812d637ab0e4 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Mon, 16 Oct 2017 10:45:33 -0700 Subject: [PATCH 019/573] Add freeze_graph to CONSOLE_SCRIPTS. (#13739) Make freeze_graph accessible from command line from pip package. --- tensorflow/tools/pip_package/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 2ffaf7b1aa..02723f3e79 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -67,6 +67,7 @@ if sys.version_info < (3, 4): # pylint: disable=line-too-long CONSOLE_SCRIPTS = [ + 'freeze_graph = tensorflow.python.tools.freeze_graph:main', 'saved_model_cli = tensorflow.python.tools.saved_model_cli:main', # We need to keep the TensorBoard command, even though the console script # is now declared by the tensorboard pip package. If we remove the -- GitLab From 531b66789bd25291266aba1fc7f7d33ece7089ce Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 10:53:09 -0700 Subject: [PATCH 020/573] Added a cleaner mechanism to set the global constants in fisher_blocks.py and fisher_factors.py in the form of a function "set_global_constants". The old way of just manually setting these constants by importing the specific modules and accessing them directly should still work, but this new method is preferred. PiperOrigin-RevId: 172345996 --- .../contrib/kfac/python/ops/fisher_blocks.py | 8 +++++++ .../kfac/python/ops/fisher_blocks_lib.py | 3 ++- .../contrib/kfac/python/ops/fisher_factors.py | 22 ++++++++++++++++--- .../kfac/python/ops/fisher_factors_lib.py | 1 + 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 3bae45b324..9d8bb8c8ce 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -34,6 +34,14 @@ from tensorflow.python.ops import math_ops NORMALIZE_DAMPING_POWER = 1.0 +def set_global_constants(normalize_damping_power=None): + """Sets various global constants used by the classes in this module.""" + global NORMALIZE_DAMPING_POWER + + if normalize_damping_power is not None: + NORMALIZE_DAMPING_POWER = normalize_damping_power + + @six.add_metaclass(abc.ABCMeta) class FisherBlock(object): """Abstract base class for objects modeling approximate Fisher matrix blocks. diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py index c6cc169b37..59389f8d38 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py @@ -31,7 +31,8 @@ _allowed_symbols = [ 'KroneckerProductFB', 'FullyConnectedKFACBasicFB', 'ConvKFCBasicFB', - 'ConvDiagonalFB' + 'ConvDiagonalFB', + 'set_global_constants', ] remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index eacd9f53b1..d3c783ee2f 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -33,9 +33,6 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.training import moving_averages -# TODO(someone): come up with a better mechanism to set these constants -# externally. See b/67084987 - # Whether to initialize covariance estimators at a zero matrix (or the identity # matrix). INIT_COVARIANCES_AT_ZERO = False @@ -53,6 +50,25 @@ EIGENVALUE_DECOMPOSITION_THRESHOLD = 2 EIGENVALUE_CLIPPING_THRESHOLD = 0.0 +def set_global_constants(init_covariances_at_zero=None, zero_debias=None, + eigenvalue_decomposition_threshold=None, + eigenvalue_clipping_threshold=None): + """Sets various global constants used by the classes in this module.""" + global INIT_COVARIANCES_AT_ZERO + global ZERO_DEBIAS + global EIGENVALUE_DECOMPOSITION_THRESHOLD + global EIGENVALUE_CLIPPING_THRESHOLD + + if init_covariances_at_zero is not None: + INIT_COVARIANCES_AT_ZERO = init_covariances_at_zero + if zero_debias is not None: + ZERO_DEBIAS = zero_debias + if eigenvalue_decomposition_threshold is not None: + EIGENVALUE_DECOMPOSITION_THRESHOLD = eigenvalue_decomposition_threshold + if eigenvalue_clipping_threshold is not None: + EIGENVALUE_CLIPPING_THRESHOLD = eigenvalue_clipping_threshold + + def inverse_initializer(shape, dtype, partition_info=None): # pylint: disable=unused-argument return array_ops.diag(array_ops.ones(shape[0], dtype)) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py b/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py index 49a07b1598..23ee93cd40 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py @@ -40,6 +40,7 @@ _allowed_symbols = [ "ConvInputKroneckerFactor", "ConvOutputKroneckerFactor", "ConvDiagonalFactor", + "set_global_constants", ] remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) -- GitLab From f8471a8012e823795f5d29f728f36e3e02dbb353 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 10:55:01 -0700 Subject: [PATCH 021/573] Fix xla_jit_compiled_cpu_function deps to pull in cpu_plugin. The intention was always for the user to only depend on xla_jit_compiled_cpu_function, and not need dependencies on internal targets. PiperOrigin-RevId: 172346257 --- tensorflow/compiler/tf2xla/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index 7865f16e53..3c94bcafc1 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -87,6 +87,7 @@ cc_library( "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:client_library", "//tensorflow/compiler/xla/client:local_client", + "//tensorflow/compiler/xla/service:cpu_plugin", "//tensorflow/compiler/xla/service/cpu:cpu_executable", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", @@ -227,7 +228,6 @@ tf_cc_test( "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:local_client", - "//tensorflow/compiler/xla/service:cpu_plugin", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", -- GitLab From cc5268be7d251e5116229f83aacab80ae6dd917f Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Mon, 16 Oct 2017 10:57:04 -0700 Subject: [PATCH 022/573] [tf.data] Fix broken implementation of `Dataset.from_generator()` on Windows. Due to a mix-up between NumPy's default array element type for a Python `int` on Windows and Linux, a tf.py_func() in `Dataset.from_generator()` would appear to return the wrong type on Windows (np.int32 instead of np.int64). All code using `Dataset.from_generator()` on Windows was previously broken. This change fixes both `tf.data.Dataset.from_generator()` and `tf.contrib.data.Dataset.from_generator()`. It also enables test coverage for this method on Windows, which should prevent future breakage. PiperOrigin-RevId: 172346533 --- tensorflow/contrib/cmake/tf_tests.cmake | 1 - .../contrib/data/python/ops/dataset_ops.py | 123 +------- tensorflow/python/data/ops/dataset_ops.py | 5 +- tensorflow/python/kernel_tests/BUILD | 21 ++ .../dataset_constructor_op_test.py | 265 +--------------- .../dataset_from_generator_op_test.py | 286 ++++++++++++++++++ 6 files changed, 321 insertions(+), 380 deletions(-) create mode 100644 tensorflow/python/kernel_tests/dataset_from_generator_op_test.py diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index a560807fb6..1d58b1d416 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -255,7 +255,6 @@ if (tensorflow_BUILD_PYTHON_TESTS) # Dataset tests "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py" # b/67743142 # Broken tensorboard test due to cmake issues. "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py" # Needs portpicker "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py" # b/65430561 diff --git a/tensorflow/contrib/data/python/ops/dataset_ops.py b/tensorflow/contrib/data/python/ops/dataset_ops.py index fe1d50db33..45d6dbe743 100644 --- a/tensorflow/contrib/data/python/ops/dataset_ops.py +++ b/tensorflow/contrib/data/python/ops/dataset_ops.py @@ -24,11 +24,8 @@ from tensorflow.contrib.data.python.ops import grouping from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gen_io_ops -from tensorflow.python.ops import script_ops from tensorflow.python.util import deprecation @@ -139,124 +136,8 @@ class Dataset(dataset_ops.Dataset): Returns: A `Dataset`. """ - if not callable(generator): - raise TypeError("`generator` must be callable.") - if output_shapes is None: - output_shapes = nest.map_structure( - lambda _: tensor_shape.TensorShape(None), output_types) - else: - output_shapes = nest.map_structure_up_to( - output_types, tensor_shape.as_shape, output_shapes) - - flattened_types = nest.flatten(output_types) - flattened_shapes = nest.flatten(output_shapes) - - generator_state = dataset_ops.Dataset._GeneratorState(generator) - - def get_iterator_id_map_fn(unused_dummy): - """Creates a unique `iterator_id` for each pass over the dataset. - - The "iterator_id" disambiguates between multiple concurrently - existing iterators. - - Args: - unused_dummy: Ignored value. - - Returns: - A `tf.int64` tensor whose value uniquely identifies an iterator in - `generator_state`. - """ - return script_ops.py_func( - generator_state.get_next_id, [], dtypes.int64, stateful=True) - - def generator_map_fn(iterator_id_t): - """Generates the next element from iterator with ID `iterator_id_t`. - - We map this function across an infinite repetition of the - `iterator_id_t`, and raise `StopIteration` to terminate the iteration. - - Args: - iterator_id_t: A `tf.int64` tensor whose value uniquely identifies - the iterator in `generator_state` from which to generate an element. - - Returns: - A nested structure of tensors representing an element from the iterator. - """ - - def generator_py_func(iterator_id): - """A `py_func` that will be called to invoke the iterator.""" - try: - values = next(generator_state.get_iterator(iterator_id)) - except StopIteration: - generator_state.iterator_completed(iterator_id) - raise StopIteration("Iteration finished.") - - # Use the same _convert function from the py_func() implementation to - # convert the returned values to arrays early, so that we can inspect - # their values. - # pylint: disable=protected-access - ret_arrays = [ - script_ops.FuncRegistry._convert(ret, dtype=dtype.as_numpy_dtype) - for ret, dtype in zip(nest.flatten_up_to(output_types, values), - flattened_types) - ] - # pylint: enable=protected-access - - # Additional type and shape checking to ensure that the components - # of the generated element match the `output_types` and `output_shapes` - # arguments. - for (ret_array, expected_dtype, expected_shape) in zip( - ret_arrays, flattened_types, flattened_shapes): - if ret_array.dtype != expected_dtype.as_numpy_dtype: - raise TypeError( - "`generator` yielded an element of type %s where an element " - "of type %s was expected." % (ret_array.dtype, - expected_dtype.as_numpy_dtype)) - if not expected_shape.is_compatible_with(ret_array.shape): - raise ValueError( - "`generator` yielded an element of shape %s where an element " - "of shape %s was expected." % (ret_array.shape, expected_shape)) - - return ret_arrays - - flat_values = script_ops.py_func( - generator_py_func, [iterator_id_t], flattened_types, stateful=True) - - # The `py_func()` op drops the inferred shapes, so we add them back in - # here. - if output_shapes is not None: - for ret_t, shape in zip(flat_values, flattened_shapes): - ret_t.set_shape(shape) - - return nest.pack_sequence_as(output_types, flat_values) - - # This function associates each traversal of `generator` with a unique - # iterator ID. - def flat_map_fn(iterator_id_t): - # First, generate an infinite dataset containing the iterator ID repeated - # forever. - repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None) - - # The `generator_map_fn` gets the next element from the iterator with the - # relevant ID, and raises StopIteration when that iterator contains no - # more elements. - return repeated_id.map(generator_map_fn) - - # A single-element dataset that, each time it is evaluated, contains a - # freshly-generated and unique (for the returned dataset) int64 - # ID that will be used to identify the appropriate Python state, which - # is encapsulated in `generator_state`, and captured in - # `get_iterator_id_map_fn`. - dummy = 0 - id_dataset = Dataset.from_tensors(dummy).map(get_iterator_id_map_fn) - - # A dataset that contains all of the elements generated by a - # single iterator created from `generator`, identified by the - # iterator ID contained in `id_dataset`. Lifting the iteration - # into a flat_map here enables multiple repetitions and/or nested - # versions of the returned dataset to be created, because it forces - # the generation of a new ID for each version. - return id_dataset.flat_map(flat_map_fn) + return Dataset(dataset_ops.Dataset.from_generator( + generator, output_types, output_shapes)) @staticmethod @deprecation.deprecated(None, "Use `tf.data.Dataset.range()`.") diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 9ea6a2cf8e..5f2e6296a8 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -201,7 +201,10 @@ class Dataset(object): with self._lock: ret = self._next_id self._next_id += 1 - return ret + # NOTE(mrry): Explicitly create an array of `np.int64` because implicit + # casting in `py_func()` will create an array of `np.int32` on Windows, + # leading to a runtime error. + return np.array(ret, dtype=np.int64) def get_iterator(self, iterator_id): return self._iterators[iterator_id] diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index d6eba3c31a..1380ef5b6a 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2890,6 +2890,27 @@ tf_py_test( ], ) +tf_py_test( + name = "dataset_from_generator_op_test", + size = "small", + srcs = ["dataset_from_generator_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:session", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + ], +) + tf_py_test( name = "filter_dataset_op_test", size = "small", diff --git a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py index 0dcce727a3..b51d483b5b 100644 --- a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py +++ b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py @@ -17,8 +17,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import threading - import numpy as np from tensorflow.core.protobuf import config_pb2 @@ -153,8 +151,9 @@ class DatasetConstructorTest(test.TestCase): # pylint: disable=g-long-lambda,unnecessary-lambda def testNestedStructure(self): - components = (np.array([1, 2, 3]), (np.array([4., 5.]), np.array([6., 7.])), - np.array([8, 9, 10])) + components = (np.array([1, 2, 3], dtype=np.int64), + (np.array([4., 5.]), np.array([6., 7.])), + np.array([8, 9, 10], dtype=np.int64)) dataset = dataset_ops.Dataset.from_tensors(components) self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), @@ -227,8 +226,10 @@ class DatasetConstructorTest(test.TestCase): # Define a separate set of components with matching leading # dimension for the from-slices constructor. - components_for_slices = (np.array([1, 2, 3]), (np.array( - [4., 5., 6.]), np.array([7., 8., 9.])), np.array([10, 11, 12])) + components_for_slices = (np.array([1, 2, 3], dtype=np.int64), + (np.array([4., 5., 6.]), + np.array([7., 8., 9.])), + np.array([10, 11, 12], dtype=np.int64)) dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices) self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), @@ -246,7 +247,7 @@ class DatasetConstructorTest(test.TestCase): self.assertEquals([3], dataset.output_shapes["b"]) def testNonSequenceNestedStructure(self): - components = np.array([1, 2, 3]) + components = np.array([1, 2, 3], dtype=np.int64) dataset = dataset_ops.Dataset.from_tensors(components) self.assertEquals(dtypes.int64, dataset.output_types) @@ -271,256 +272,6 @@ class DatasetConstructorTest(test.TestCase): self.assertEquals(dtypes.int64, get_next.dtype) self.assertEquals([3], get_next.shape) - def _testFromGenerator(self, generator, elem_sequence, num_repeats): - iterator = ( - dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) - .repeat(num_repeats) - .prefetch(5) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - for _ in range(2): # Run twice to test reinitialization. - sess.run(init_op) - for _ in range(num_repeats): - for elem in elem_sequence: - self.assertAllEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats): - iterator = ( - dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) - .repeat(num_repeats) - .prefetch(5) - .make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.test_session() as sess: - for _ in range(num_repeats): - for elem in elem_sequence: - self.assertAllEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorUsingFunction(self): - def generator(): - for i in range(1, 100): - yield [i] * i - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - self._testFromGeneratorOneShot(generator, elem_sequence, 1) - self._testFromGeneratorOneShot(generator, elem_sequence, 5) - - def testFromGeneratorUsingList(self): - generator = lambda: [[i] * i for i in range(1, 100)] - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - - def testFromGeneratorUsingNdarray(self): - generator = lambda: np.arange(100, dtype=np.int64) - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - - def testFromGeneratorUsingGeneratorExpression(self): - # NOTE(mrry): Generator *expressions* are not repeatable (or in - # general reusable), because they eagerly evaluate the `for` - # expression as `iter(range(1, 100))` and discard the means of - # reconstructing `range(1, 100)`. Wrapping the generator - # expression in a `lambda` makes it repeatable. - generator = lambda: ([i] * i for i in range(1, 100)) - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - - def testFromMultipleConcurrentGenerators(self): - num_inner_repeats = 5 - num_outer_repeats = 100 - - def generator(): - for i in range(1, 10): - yield ([i] * i, [i, i ** 2, i ** 3]) - input_list = list(generator()) - - # The interleave transformation is essentially a flat map that - # draws from multiple input datasets concurrently (in a cyclic - # fashion). By placing `Datsaet.from_generator()` inside an - # interleave, we test its behavior when multiple iterators are - # active at the same time; by additionally prefetching inside the - # interleave, we create the possibility of parallel (modulo GIL) - # invocations to several iterators created by the same dataset. - def interleave_fn(_): - return (dataset_ops.Dataset.from_generator( - generator, output_types=(dtypes.int64, dtypes.int64), - output_shapes=([None], [3])) - .repeat(num_inner_repeats).prefetch(5)) - - iterator = ( - dataset_ops.Dataset.range(num_outer_repeats) - .interleave(interleave_fn, cycle_length=10, - block_length=len(input_list)) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - for _ in range(num_inner_repeats * num_outer_repeats): - for elem in input_list: - val0, val1 = sess.run(get_next) - self.assertAllEqual(elem[0], val0) - self.assertAllEqual(elem[1], val1) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorsRunningInParallel(self): - num_parallel_iterators = 3 - - # Define shared state that multiple iterator instances will access to - # demonstrate their concurrent activity. - lock = threading.Lock() - condition = threading.Condition(lock) - next_ticket = [0] # GUARDED_BY(lock) - - def generator(): - # NOTE(mrry): We yield one element before the barrier, because - # the current implementation of `Dataset.interleave()` must - # fetch one element from each incoming dataset to start the - # prefetching. - yield 0 - - # Define a barrier that `num_parallel_iterators` iterators must enter - # before any can proceed. Demonstrates that multiple iterators may be - # active at the same time. - condition.acquire() - ticket = next_ticket[0] - next_ticket[0] += 1 - if ticket == num_parallel_iterators - 1: - # The last iterator to join the barrier notifies the others. - condition.notify_all() - else: - # Wait until the last iterator enters the barrier. - while next_ticket[0] < num_parallel_iterators: - condition.wait() - condition.release() - - yield 1 - - # As in `testFromMultipleConcurrentGenerators()`, we use a combination of - # `Dataset.interleave()` and `Dataset.prefetch()` to cause multiple - # iterators to be active concurrently. - def interleave_fn(_): - return dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2) - - iterator = ( - dataset_ops.Dataset.range(num_parallel_iterators) - .interleave( - interleave_fn, cycle_length=num_parallel_iterators, block_length=1) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - for elem in [0, 1]: - for _ in range(num_parallel_iterators): - self.assertAllEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorImplicitConversion(self): - def generator(): - yield [1] - yield [2] - yield [3] - - for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]: - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtype, output_shapes=[1]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual(dtype, get_next.dtype) - - with self.test_session() as sess: - sess.run(init_op) - for expected in [[1], [2], [3]]: - next_val = sess.run(get_next) - self.assertEqual(dtype.as_numpy_dtype, next_val.dtype) - self.assertAllEqual(expected, next_val) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorTypeError(self): - def generator(): - yield np.array([1, 2, 3], dtype=np.int64) - yield np.array([4, 5, 6], dtype=np.int64) - yield "ERROR" - yield np.array([7, 8, 9], dtype=np.int64) - - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[3]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - self.assertAllEqual([1, 2, 3], sess.run(get_next)) - self.assertAllEqual([4, 5, 6], sess.run(get_next)) - with self.assertRaisesOpError(r"invalid literal for long\(\)"): - sess.run(get_next) - self.assertAllEqual([7, 8, 9], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorShapeError(self): - def generator(): - yield np.array([1, 2, 3], dtype=np.int64) - yield np.array([4, 5, 6], dtype=np.int64) - yield np.array([7, 8, 9, 10], dtype=np.int64) - yield np.array([11, 12, 13], dtype=np.int64) - - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[3]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - self.assertAllEqual([1, 2, 3], sess.run(get_next)) - self.assertAllEqual([4, 5, 6], sess.run(get_next)) - with self.assertRaisesOpError(r"element of shape \(3,\) was expected"): - sess.run(get_next) - self.assertAllEqual([11, 12, 13], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorHeterogeneous(self): - def generator(): - yield 1 - yield [2, 3] - - iterator = ( - dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - self.assertAllEqual(1, sess.run(get_next)) - self.assertAllEqual([2, 3], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - def testSplitPipelineFailsWithPlacementError(self): with session.Session( target="", diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py new file mode 100644 index 0000000000..e774256695 --- /dev/null +++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py @@ -0,0 +1,286 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the experimental input pipeline ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import threading + +import numpy as np + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.platform import test + + +class DatasetConstructorTest(test.TestCase): + + def _testFromGenerator(self, generator, elem_sequence, num_repeats): + iterator = ( + dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) + .repeat(num_repeats) + .prefetch(5) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + for _ in range(2): # Run twice to test reinitialization. + sess.run(init_op) + for _ in range(num_repeats): + for elem in elem_sequence: + self.assertAllEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats): + iterator = ( + dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) + .repeat(num_repeats) + .prefetch(5) + .make_one_shot_iterator()) + get_next = iterator.get_next() + + with self.test_session() as sess: + for _ in range(num_repeats): + for elem in elem_sequence: + self.assertAllEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorUsingFunction(self): + def generator(): + for i in range(1, 100): + yield [i] * i + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + self._testFromGeneratorOneShot(generator, elem_sequence, 1) + self._testFromGeneratorOneShot(generator, elem_sequence, 5) + + def testFromGeneratorUsingList(self): + generator = lambda: [[i] * i for i in range(1, 100)] + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + + def testFromGeneratorUsingNdarray(self): + generator = lambda: np.arange(100, dtype=np.int64) + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + + def testFromGeneratorUsingGeneratorExpression(self): + # NOTE(mrry): Generator *expressions* are not repeatable (or in + # general reusable), because they eagerly evaluate the `for` + # expression as `iter(range(1, 100))` and discard the means of + # reconstructing `range(1, 100)`. Wrapping the generator + # expression in a `lambda` makes it repeatable. + generator = lambda: ([i] * i for i in range(1, 100)) + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + + def testFromMultipleConcurrentGenerators(self): + num_inner_repeats = 5 + num_outer_repeats = 100 + + def generator(): + for i in range(1, 10): + yield ([i] * i, [i, i ** 2, i ** 3]) + input_list = list(generator()) + + # The interleave transformation is essentially a flat map that + # draws from multiple input datasets concurrently (in a cyclic + # fashion). By placing `Datsaet.from_generator()` inside an + # interleave, we test its behavior when multiple iterators are + # active at the same time; by additionally prefetching inside the + # interleave, we create the possibility of parallel (modulo GIL) + # invocations to several iterators created by the same dataset. + def interleave_fn(_): + return (dataset_ops.Dataset.from_generator( + generator, output_types=(dtypes.int64, dtypes.int64), + output_shapes=([None], [3])) + .repeat(num_inner_repeats).prefetch(5)) + + iterator = ( + dataset_ops.Dataset.range(num_outer_repeats) + .interleave(interleave_fn, cycle_length=10, + block_length=len(input_list)) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for _ in range(num_inner_repeats * num_outer_repeats): + for elem in input_list: + val0, val1 = sess.run(get_next) + self.assertAllEqual(elem[0], val0) + self.assertAllEqual(elem[1], val1) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorsRunningInParallel(self): + num_parallel_iterators = 3 + + # Define shared state that multiple iterator instances will access to + # demonstrate their concurrent activity. + lock = threading.Lock() + condition = threading.Condition(lock) + next_ticket = [0] # GUARDED_BY(lock) + + def generator(): + # NOTE(mrry): We yield one element before the barrier, because + # the current implementation of `Dataset.interleave()` must + # fetch one element from each incoming dataset to start the + # prefetching. + yield 0 + + # Define a barrier that `num_parallel_iterators` iterators must enter + # before any can proceed. Demonstrates that multiple iterators may be + # active at the same time. + condition.acquire() + ticket = next_ticket[0] + next_ticket[0] += 1 + if ticket == num_parallel_iterators - 1: + # The last iterator to join the barrier notifies the others. + condition.notify_all() + else: + # Wait until the last iterator enters the barrier. + while next_ticket[0] < num_parallel_iterators: + condition.wait() + condition.release() + + yield 1 + + # As in `testFromMultipleConcurrentGenerators()`, we use a combination of + # `Dataset.interleave()` and `Dataset.prefetch()` to cause multiple + # iterators to be active concurrently. + def interleave_fn(_): + return dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2) + + iterator = ( + dataset_ops.Dataset.range(num_parallel_iterators) + .interleave( + interleave_fn, cycle_length=num_parallel_iterators, block_length=1) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for elem in [0, 1]: + for _ in range(num_parallel_iterators): + self.assertAllEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorImplicitConversion(self): + def generator(): + yield [1] + yield [2] + yield [3] + + for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]: + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtype, output_shapes=[1]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + self.assertEqual(dtype, get_next.dtype) + + with self.test_session() as sess: + sess.run(init_op) + for expected in [[1], [2], [3]]: + next_val = sess.run(get_next) + self.assertEqual(dtype.as_numpy_dtype, next_val.dtype) + self.assertAllEqual(expected, next_val) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorTypeError(self): + def generator(): + yield np.array([1, 2, 3], dtype=np.int64) + yield np.array([4, 5, 6], dtype=np.int64) + yield "ERROR" + yield np.array([7, 8, 9], dtype=np.int64) + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[3]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + self.assertAllEqual([1, 2, 3], sess.run(get_next)) + self.assertAllEqual([4, 5, 6], sess.run(get_next)) + # NOTE(mrry): Type name in message differs between Python 2 (`long`) and + # 3 (`int`). + with self.assertRaisesOpError(r"invalid literal for"): + sess.run(get_next) + self.assertAllEqual([7, 8, 9], sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorShapeError(self): + def generator(): + yield np.array([1, 2, 3], dtype=np.int64) + yield np.array([4, 5, 6], dtype=np.int64) + yield np.array([7, 8, 9, 10], dtype=np.int64) + yield np.array([11, 12, 13], dtype=np.int64) + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[3]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + self.assertAllEqual([1, 2, 3], sess.run(get_next)) + self.assertAllEqual([4, 5, 6], sess.run(get_next)) + with self.assertRaisesOpError(r"element of shape \(3,\) was expected"): + sess.run(get_next) + self.assertAllEqual([11, 12, 13], sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorHeterogeneous(self): + def generator(): + yield 1 + yield [2, 3] + + iterator = ( + dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64).make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + self.assertAllEqual(1, sess.run(get_next)) + self.assertAllEqual([2, 3], sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + +if __name__ == "__main__": + test.main() -- GitLab From f0e3edf8b1c8de49672d78abe73dcd0b1f02620c Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Mon, 16 Oct 2017 11:09:36 -0700 Subject: [PATCH 023/573] [TF2XLA] Keep Switch and Merge nodes in own clusters. * Keep Switch and Merge nodes in separate clusters to avoid creating irreducible graphs; * Merge Switch nodes with common predicates; * Add support for if-then structure; * Squash trivial Switch->Merge groups; * Merge newly Merge free nodes with Switch & Merge free inputs; * Check to see if it is a Merge node before merging to common merge node; * Return an error if all Switches have not been replaced; * Add test fir tf,case; PiperOrigin-RevId: 172348729 --- .../tf2xla/functionalize_control_flow.cc | 233 ++++++++++++++---- 1 file changed, 181 insertions(+), 52 deletions(-) diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc index 40bc164c50..abfc856904 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc @@ -74,6 +74,18 @@ struct Frame { std::unordered_set nodes; }; +// Returns a textual representation of the names of the nodes in the input. +template +string NodesToString(const T& nodes) { + return strings::StrCat("{", + str_util::Join(nodes, ",", + [](string* output, const Node* node) { + strings::StrAppend(output, + node->name()); + }), + "}"); +} + // Copies a subgraph from `graph` to `output` by performing a reverse DFS // starting at nodes in vector `stack`. // `node_map` is a vector indexed by source node ID to dest nodes. @@ -93,12 +105,13 @@ Status CopySubgraph(const Graph& graph, const Frame* frame, std::vector stack, const std::vector& squash_src_outputs, std::vector* node_map, Graph* output) { + VLOG(3) << "Stack: " << NodesToString(stack); std::vector visited(graph.num_node_ids(), false); while (!stack.empty()) { Node* n = stack.back(); stack.pop_back(); - VLOG(3) << "Copying node " << n->name(); + VLOG(5) << "Copying node " << n->name(); if (visited[n->id()]) continue; visited[n->id()] = true; @@ -577,7 +590,7 @@ class FunctionalizeCond { // id in the original graph. struct CondArgs { struct CondCmp { - bool operator()(const Node* a, const Node* b) { + bool operator()(const Node* a, const Node* b) const { return a->id() < b->id(); } }; @@ -613,7 +626,10 @@ class FunctionalizeCond { // If `from` and `to` correspond to different clusters, then merge the nodes // in the clustered graph corresponding to `from` and `to`. - void ContractEdge(Cluster* from, Cluster* to); + // + // If `remove_from_graph` is specified then the `from` node is also removed + // from the clustered graph post contracting the edge. + void ContractEdge(Cluster* from, Cluster* to, bool remove_from_graph = false); // Converts a Merge node to a XlaIf. This encapsulates the process of // extracting the bodies needed for the then and else branch, creates a XlaIf @@ -621,6 +637,10 @@ class FunctionalizeCond { // merge node with a XlaIf. Status ConvertMergeToXlaIf(Cluster* merge_cluster); + // Removes a Switch cluster feeding directly into a Merge cluster by removing + // the Switch and Merge nodes and collapsing into a single cluster. + Status RemoveTrivialMerge(Cluster* merge_cluster); + // Returns the switch cluster corresponding to the merge node. This function // only returns the switch cluster in the simple case where we have a switch // node is the entry of a diamond corresponding to a conditional: @@ -629,7 +649,10 @@ class FunctionalizeCond { // / \ // Branch Branch // \ / - // merge_cluster + // merge_cluster + // + // Note: either of the branches may be empty. The case where both branches are + // empty is handled by RemoveTrivialMerge. gtl::optional GetSwitchCluster(const Cluster& merge_cluster); // Determines the arguments needed as input to the Merge cluster originating @@ -661,8 +684,8 @@ class FunctionalizeCond { template void RemoveUnusedArgs(const T& args); - // Removes all Merge nodes that are unused. - void RemoveUnusedMergeNodes(Cluster* merge_cluster); + // Removes all Merge nodes in merge_cluster. + void RemoveMergeNodes(Cluster* merge_cluster); // Returns the representative member of the corresponding cluster. ClusterHandle Representative(const Node* node) { @@ -713,6 +736,24 @@ string DebugString(const Graph& graph, return strings::StrCat(ret, "}"); } +string DebugString(const FunctionalizeCond::ClusteredGraph& clustered_graph) { + string ret = "digraph {\ncompound=true;labeljust=\"r\";\n"; + auto name = [](const FunctionalizeCond::Cluster& cluster) { + return cluster.representative.ToString(); + }; + for (auto kv : clustered_graph) { + strings::StrAppend(&ret, kv.first.ToString(), " [label=\"", name(kv.second), + " (", kv.second.switch_nodes.size(), ", ", + kv.second.merge_nodes.size(), ")\"];\n"); + } + for (auto kv : clustered_graph) { + for (auto in : kv.second.in_nodes) { + strings::StrAppend(&ret, name(*in), " -> ", name(kv.second), ";\n"); + } + } + return strings::StrCat(ret, "}"); +} + bool IsDeadSwitch(const Node* node) { for (const Edge* e : node->out_edges()) { const Node* dst = e->dst(); @@ -754,21 +795,22 @@ void FunctionalizeCond::CreateClusters() { // conservatively assuming all merge nodes become XlaIf nodes. clusters_.resize(clusters_.size() + merge_nodes_.size()); - // Merge a cluster with its input, unless the input is a Switch node or the - // node is a Merge node. + // Merge a cluster with its input, unless the input is a Switch node or + // the node is a Merge node. for (const Node* node : graph_->nodes()) { - if (IsMerge(node) || !node->IsOp()) { + if (IsMerge(node) || IsSwitch(node) || !node->IsOp()) { continue; } for (const Node* in : node->in_nodes()) { - if (!IsSwitch(in) && in->IsOp()) { + if (in->IsOp() && !IsSwitch(in) && !IsMerge(in)) { clusters_.at(node).Merge(&clusters_.at(in)); } } } } -void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) { +void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to, + bool remove_from_graph) { VLOG(3) << "ContractEdge from = " << from->representative << " to = " << to->representative; if (from->representative == to->representative) { @@ -801,6 +843,10 @@ void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) { to->out_nodes.erase(from); clusters_.at(to->representative).Merge(&clusters_.at(from->representative)); from->visited = true; + + if (remove_from_graph) { + clustered_graph_.erase(from->representative); + } } void FunctionalizeCond::CreateClusteredGraph() { @@ -839,6 +885,22 @@ void FunctionalizeCond::CreateClusteredGraph() { update_cluster_for_node(node).merge_nodes.insert(node); } + // Merge Switch nodes with common predicate. + std::unordered_map> predicate_to_switch; + for (Node* node : switch_nodes_) { + Node* tmp; + TF_CHECK_OK(node->input_node(1, &tmp)); + predicate_to_switch[tmp].push_back(node); + } + for (auto kv : predicate_to_switch) { + Cluster& first = clustered_graph_.at(Representative(kv.second.front())); + for (Node* switch_node : kv.second) { + ClusterHandle handle = Representative(switch_node); + Cluster& cluster = clustered_graph_.at(handle); + ContractEdge(&cluster, &first, /*remove_from_graph=*/true); + } + } + // Merge Merge nodes with common input together. for (Node* node : merge_nodes_) { Cluster& cluster = clustered_graph_.at(Representative(node)); @@ -847,35 +909,47 @@ void FunctionalizeCond::CreateClusteredGraph() { continue; } Cluster& cluster_node_in = clustered_graph_.at(Representative(in)); + // ContractEdge can modify out_nodes of cluster_node_in, so traverse + // over out_nodes assuming it does. for (auto it = cluster_node_in.out_nodes.begin(); it != cluster_node_in.out_nodes.end();) { - ContractEdge(*it++, &cluster); + if (!(*it)->merge_nodes.empty()) { + ContractEdge(*it++, &cluster, /*remove_from_graph=*/true); + } else { + ++it; + } } } } - VLOG(3) << "ClusteredGraph: " << DebugString(*graph_, &clusters_); + VLOG(3) << "Graph with clusters: " << DebugString(*graph_, &clusters_); + VLOG(3) << "ClusteredGraph: " << DebugString(clustered_graph_); } gtl::optional FunctionalizeCond::GetSwitchCluster( const Cluster& merge_cluster) { VLOG(3) << "GetSwitchCluster for " << merge_cluster.representative; gtl::optional switch_cluster; - if (merge_cluster.in_nodes.size() != 2) { + if (merge_cluster.in_nodes.size() > 2) { return gtl::nullopt; } - for (const Cluster* in : merge_cluster.in_nodes) { - if (in->in_nodes.size() != 1) { + for (Cluster* in : merge_cluster.in_nodes) { + Cluster* cluster = in; + if (in->switch_nodes.empty()) { + if (in->in_nodes.size() != 1) { + return gtl::nullopt; + } + // There is only a single `in` cluster. + cluster = *in->in_nodes.begin(); + } + if (cluster->switch_nodes.empty()) { return gtl::nullopt; } - for (auto inin : in->in_nodes) { - if (switch_cluster.has_value()) { - if (*switch_cluster != inin) { - return gtl::nullopt; - } - } else { - switch_cluster = inin; - } + + if (switch_cluster.has_value() && *switch_cluster != cluster) { + return gtl::nullopt; + } else { + switch_cluster = cluster; } } return switch_cluster; @@ -889,6 +963,9 @@ xla::StatusOr FunctionalizeCond::DetermineCondArgs( auto feeds_into_branch_cluster = [&](Node* switch_cluster) { for (Node* out : switch_cluster->out_nodes()) { ClusterHandle repr = Representative(out); + if (repr == merge_cluster.representative) { + return true; + } for (Cluster* in : merge_cluster.in_nodes) { if (repr == in->representative) { return true; @@ -919,12 +996,9 @@ xla::StatusOr FunctionalizeCond::DetermineCondArgs( xla::StatusOr FunctionalizeCond::BuildAndAddXlaIfOp( const CondArgs& cond_args, const Cluster& merge_cluster, const std::vector& outputs) { - VLOG(2) << "Build if op for {" - << str_util::Join(merge_cluster.merge_nodes, ", ", - [](string* out, const Node* node) { - strings::StrAppend(out, node->name()); - }) - << "}"; + VLOG(2) << "Build if op for " << NodesToString(merge_cluster.merge_nodes) + << " with input " << NodesToString(cond_args.args); + NodeDef if_def; // Create a new If node using the name of the merge node. NodeDefBuilder builder( @@ -941,6 +1015,7 @@ xla::StatusOr FunctionalizeCond::BuildAndAddXlaIfOp( auto body = xla::MakeUnique(graph_->op_registry()); TF_RETURN_IF_ERROR( ExtractBody(cond_args, merge_cluster, outputs, i, body.get())); + VLOG(3) << "Body " << branch[i] << ": " << DebugString(body.get()); FunctionDef body_fdef; TF_RETURN_IF_ERROR(GraphToFunctionDef(*body, body_name.name(), &body_fdef)); TF_RETURN_IF_ERROR(library_->AddFunctionDef(body_fdef)); @@ -1001,10 +1076,7 @@ void FunctionalizeCond::RemoveClusterNodes(Cluster* cluster) { template void FunctionalizeCond::RemoveUnusedArgs(const T& args) { - VLOG(2) << "RemoveUnusedArgs among: " - << str_util::Join(args, ", ", [](string* output, const Node* node) { - strings::StrAppend(output, node->name()); - }); + VLOG(2) << "RemoveUnusedArgs among: " << NodesToString(args); std::deque to_delete; for (Node* arg : args) { @@ -1029,7 +1101,8 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args, const Cluster& merge_cluster, const std::vector& outputs, int input_edge, Graph* body) { - VLOG(2) << "ExtractBody for " << merge_cluster.representative; + VLOG(2) << "ExtractBody for " << merge_cluster.representative + << " along edge " << input_edge; std::vector squash_src_outputs(graph_->num_node_ids(), false); std::vector node_map(graph_->num_node_ids(), nullptr); int arg_count = 0; @@ -1053,12 +1126,21 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args, TF_ASSIGN_OR_RETURN(node_map.at(node->id()), BuildRetvalNode(body, node->output_type(0), /*index=*/j)); - Node* in; - TF_RETURN_IF_ERROR(node->input_node(input_edge, &in)); + const Edge* in_edge; + TF_RETURN_IF_ERROR(node->input_edge(input_edge, &in_edge)); + Node* in = in_edge->src(); if (node_map.at(in->id()) == nullptr) { node_map.at(in->id()) = body->CopyNode(in); } - body->AddEdge(node_map.at(in->id()), j, node_map.at(node->id()), 0); + + if (cond_args.args.find(in) == cond_args.args.end()) { + body->AddEdge(node_map.at(in->id()), in_edge->src_output(), + node_map.at(node->id()), 0); + } else { + body->AddEdge(node_map.at(in->id()), 0, node_map.at(node->id()), 0); + // Don't include input nodes that are already just returned in stack. + continue; + } stack.push_back(in); } @@ -1108,17 +1190,46 @@ Status FunctionalizeCond::AddOutputEdges(const std::vector& outputs, return Status::OK(); } -void FunctionalizeCond::RemoveUnusedMergeNodes(Cluster* merge_cluster) { - VLOG(3) << "RemoveUnusedMergeNodes for " << merge_cluster->representative; +void FunctionalizeCond::RemoveMergeNodes(Cluster* merge_cluster) { + VLOG(3) << "RemoveMergeNodes for " << merge_cluster->representative; // Remove all merge nodes now dead post extraction of If. for (auto it = merge_cluster->merge_nodes.begin(); it != merge_cluster->merge_nodes.end();) { Node* node = *it; - if (node->out_edges().empty()) { - graph_->RemoveNode(node); - merge_cluster->merge_nodes.erase(*it++); + graph_->RemoveNode(node); + merge_cluster->merge_nodes.erase(*it++); + } +} + +Status FunctionalizeCond::RemoveTrivialMerge(Cluster* merge_cluster) { + Cluster* switch_cluster = *merge_cluster->in_nodes.begin(); + if (switch_cluster->switch_nodes.empty()) { + return errors::FailedPrecondition( + "Not a trivial merge: no Switch node feeding into Merge node"); + } + + for (auto it = merge_cluster->merge_nodes.begin(); + it != merge_cluster->merge_nodes.end();) { + // We have the following structure: + // Op -> Switch -> Merge -> Consumer + // and we want to transform it to: + // Op -> Consumer + Node* merge_node = *it; + Node* switch_node; + const Edge* in = nullptr; + TF_RETURN_IF_ERROR(merge_node->input_node(0, &switch_node)); + TF_RETURN_IF_ERROR(switch_node->input_edge(0, &in)); + for (auto out : merge_node->out_edges()) { + int src_output = out->dst_input() == Graph::kControlSlot + ? Graph::kControlSlot + : in->src_output(); + graph_->AddEdge(in->src(), src_output, out->dst(), out->dst_input()); } + graph_->RemoveNode(*it++); } + RemoveUnusedArgs(switch_cluster->switch_nodes); + + return Status::OK(); } Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { @@ -1127,12 +1238,8 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { if (!switch_cluster.has_value()) { return errors::FailedPrecondition( "Merge cluster was not part of a simple conditional in the clustered " - "graph. Graph nodes in merge cluster {", - str_util::Join(merge_cluster->merge_nodes, ", ", - [](string* output, Node* node) { - strings::StrAppend(output, node->name()); - }), - "}"); + "graph. Graph nodes in merge cluster ", + NodesToString(merge_cluster->merge_nodes)); } TF_ASSIGN_OR_RETURN(auto cond_args, DetermineCondArgs(*merge_cluster, **switch_cluster)); @@ -1153,15 +1260,17 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { // Remove the old nodes from the graph_ and contract the edges of the // clustered graph. for (auto in : merge_cluster->in_nodes) { - RemoveClusterNodes(in); + if (in != *switch_cluster) { + RemoveClusterNodes(in); + } } + RemoveMergeNodes(merge_cluster); RemoveUnusedArgs(cond_args.args); auto in_nodes = merge_cluster->in_nodes; for (auto it = in_nodes.begin(); it != in_nodes.end();) { ContractEdge(*it++, merge_cluster); } ContractEdge(*switch_cluster, merge_cluster); - RemoveUnusedMergeNodes(merge_cluster); clusters_[if_node].Get() = ClusterHandle(merge_cluster->representative); return Status::OK(); @@ -1230,7 +1339,27 @@ Status FunctionalizeCond::Functionalize(Graph* graph, for (auto it = queue.begin(); it != queue.end();) { Cluster* merge_cluster = (*it).second; ++it; - TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster)); + if (merge_cluster->in_nodes.size() == 1) { + TF_RETURN_IF_ERROR(fc.RemoveTrivialMerge(merge_cluster)); + } else { + TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster)); + } + + // Contract newly Merge free merge_cluster with incoming nodes without + // Switch or Merge nodes. + std::vector in_nodes(merge_cluster->in_nodes.begin(), + merge_cluster->in_nodes.end()); + for (auto in : in_nodes) { + if (in->merge_nodes.empty() && in->switch_nodes.empty()) { + fc.ContractEdge(in, merge_cluster); + } + } + } + + if (!fc.switch_nodes_.empty()) { + return errors::Internal( + "Failed to functionalize control flow with Switch nodes remaining: ", + NodesToString(fc.switch_nodes_)); } return Status::OK(); } -- GitLab From 2487732ff111daedaf489672700ccfbf2088c3de Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 11:17:08 -0700 Subject: [PATCH 024/573] Add tf.contrib.distributions.bijectors.Gumbel. PiperOrigin-RevId: 172350038 --- tensorflow/contrib/distributions/BUILD | 19 +++ .../kernel_tests/bijectors/gumbel_test.py | 70 ++++++++++ .../python/ops/bijectors/__init__.py | 2 + .../python/ops/bijectors/gumbel.py | 29 ++++ .../python/ops/bijectors/gumbel_impl.py | 124 ++++++++++++++++++ 5 files changed, 244 insertions(+) create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index 93770c37de..825ec652d0 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -797,6 +797,25 @@ cuda_py_test( ], ) +cuda_py_test( + name = "gumbel_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/gumbel_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "@six_archive//:six", + "//tensorflow/contrib/linalg:linalg_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "inline_test", size = "small", diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py new file mode 100644 index 0000000000..9a905980c7 --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py @@ -0,0 +1,70 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +from scipy import stats + +from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import Gumbel +from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite +from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency +from tensorflow.python.platform import test + + +class GumbelBijectorTest(test.TestCase): + """Tests correctness of the Gumbel bijector.""" + + def testBijector(self): + with self.test_session(): + loc = 0.3 + scale = 5. + bijector = Gumbel(loc=loc, scale=scale, event_ndims=1, validate_args=True) + self.assertEqual("gumbel", bijector.name) + x = np.array([[[-3.], [0.], [0.5], [4.2], [12.]]], dtype=np.float32) + # Gumbel distribution + gumbel_dist = stats.gumbel_r(loc=loc, scale=scale) + y = gumbel_dist.cdf(x).astype(np.float32) + self.assertAllClose(y, bijector.forward(x).eval()) + self.assertAllClose(x, bijector.inverse(y).eval()) + self.assertAllClose( + # We should lose a dimension from calculating the determinant of the + # jacobian. + np.squeeze(gumbel_dist.logpdf(x), axis=2), + bijector.forward_log_det_jacobian(x).eval()) + self.assertAllClose( + -bijector.inverse_log_det_jacobian(y).eval(), + bijector.forward_log_det_jacobian(x).eval(), + rtol=1e-4, + atol=0.) + + def testScalarCongruency(self): + with self.test_session(): + assert_scalar_congruency( + Gumbel(loc=0.3, scale=20.), lower_x=1., upper_x=100., rtol=0.02) + + def testBijectiveAndFinite(self): + with self.test_session(): + bijector = Gumbel(loc=0., scale=3.0, event_ndims=0, validate_args=True) + x = np.linspace(-10., 10., num=10).astype(np.float32) + y = np.linspace(0.01, 0.99, num=10).astype(np.float32) + assert_bijective_and_finite(bijector, x, y, rtol=1e-3) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py index c9ed546a34..e62f900bbf 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py @@ -22,6 +22,7 @@ @@CholeskyOuterProduct @@ConditionalBijector @@Exp +@@Gumbel @@Identity @@Inline @@Invert @@ -48,6 +49,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.chain import * from tensorflow.contrib.distributions.python.ops.bijectors.cholesky_outer_product import * from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector import * from tensorflow.contrib.distributions.python.ops.bijectors.exp import * +from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import * from tensorflow.contrib.distributions.python.ops.bijectors.inline import * from tensorflow.contrib.distributions.python.ops.bijectors.invert import * from tensorflow.contrib.distributions.python.ops.bijectors.permute import * diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py new file mode 100644 index 0000000000..cf37aa5111 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py @@ -0,0 +1,29 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gumbel bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.contrib.distributions.python.ops.bijectors.gumbel_impl import * +# pylint: enable=wildcard-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = ["Gumbel"] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py new file mode 100644 index 0000000000..67f3978556 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py @@ -0,0 +1,124 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gumbel bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bijector + +__all__ = [ + "Gumbel", +] + + +class Gumbel(bijector.Bijector): + """Compute `Y = g(X) = exp(-exp(-(X - loc) / scale))`. + + This bijector maps inputs from `[-inf, inf]` to [0, 1]`. The inverse of the + bijector applied to a uniform random variable `X ~ U(0, 1) gives back a + random variable with the + [Gumbel distribution](https://en.wikipedia.org/wiki/Gumbel_distribution): + + ```none + Y ~ Gumbel(loc, scale) + pdf(y; loc, scale) = exp( + -( (y - loc) / scale + exp(- (y - loc) / scale) ) ) / scale + ``` + """ + + def __init__(self, + loc=0., + scale=1., + event_ndims=0, + validate_args=False, + name="gumbel"): + """Instantiates the `Gumbel` bijector. + + Args: + loc: Float-like `Tensor` that is the same dtype and is + broadcastable with `scale`. + This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. + scale: Positive Float-like `Tensor` that is the same dtype and is + broadcastable with `loc`. + This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. + event_ndims: Python scalar indicating the number of dimensions associated + with a particular draw from the distribution. + validate_args: Python `bool` indicating whether arguments should be + checked for correctness. + name: Python `str` name given to ops managed by this object. + """ + self._graph_parents = [] + self._name = name + self._validate_args = validate_args + with self._name_scope("init", values=[loc, scale]): + self._loc = ops.convert_to_tensor(loc, name="loc") + self._scale = ops.convert_to_tensor(scale, name="scale") + check_ops.assert_same_float_dtype([self._loc, self._scale]) + if validate_args: + self._scale = control_flow_ops.with_dependencies([ + check_ops.assert_positive( + self._scale, message="Argument scale was not positive") + ], self._scale) + + super(Gumbel, self).__init__( + event_ndims=event_ndims, validate_args=validate_args, name=name) + + @property + def loc(self): + """The `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.""" + return self._loc + + @property + def scale(self): + """This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.""" + return self._scale + + def _forward(self, x): + z = (x - self.loc) / self.scale + return math_ops.exp(-math_ops.exp(-z)) + + def _inverse(self, y): + y = self._maybe_assert_valid_y(y) + return self.loc - self.scale * math_ops.log(-math_ops.log(y)) + + def _inverse_log_det_jacobian(self, y): + y = self._maybe_assert_valid_y(y) + event_dims = self._event_dims_tensor(y) + return math_ops.reduce_sum( + math_ops.log(self.scale / (-math_ops.log(y) * y)), axis=event_dims) + + def _forward_log_det_jacobian(self, x): + event_dims = self._event_dims_tensor(x) + z = (x - self.loc) / self.scale + return math_ops.reduce_sum( + -z - math_ops.exp(-z) - math_ops.log(self.scale), axis=event_dims) + + def _maybe_assert_valid_y(self, y): + if not self.validate_args: + return y + is_positive = check_ops.assert_non_negative( + y, message="Inverse transformation input must be greater than 0.") + less_than_one = check_ops.assert_less_equal( + y, + constant_op.constant(1., y.dtype), + message="Inverse transformation input must be less than or equal to 1.") + return control_flow_ops.with_dependencies([is_positive, less_than_one], y) -- GitLab From 74bd8ff717eaf08bf64f4b16c0bca40173b19614 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Mon, 16 Oct 2017 11:32:49 -0700 Subject: [PATCH 025/573] [tf.contrib.seq2seq] Some light cleanup in beam search decoder code. PiperOrigin-RevId: 172352767 --- .../kernel_tests/beam_search_decoder_test.py | 3 +- .../seq2seq/python/ops/beam_search_decoder.py | 71 ++++++++++--------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py index 2caeb9eb61..8d4ec4b4db 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py @@ -80,8 +80,7 @@ class TestEosMasking(test.TestCase): ]) eos_token = 0 - previously_finished = constant_op.constant( - [[0, 1, 0], [0, 1, 1]], dtype=dtypes.float32) + previously_finished = np.array([[0, 1, 0], [0, 1, 1]], dtype=bool) masked = beam_search_decoder._mask_probs(probs, eos_token, previously_finished) diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py index e22912ac5c..112ac57a1b 100644 --- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py +++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py @@ -20,9 +20,10 @@ from __future__ import print_function import collections +import numpy as np + from tensorflow.contrib.seq2seq.python.ops import beam_search_ops from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -390,17 +391,17 @@ class BeamSearchDecoder(decoder.Decoder): We do this so that we can use nest and not run into problems with shapes. Args: - t: Tensor of dimension [batch_size*beam_width, s] - s: Tensor, Python int, or TensorShape. + t: `Tensor`, either scalar or shaped `[batch_size * beam_width] + s`. + s: `Tensor`, Python int, or `TensorShape`. Returns: - Either a reshaped version of t with dimension - [batch_size, beam_width, s] if t's first dimension is of size - batch_size*beam_width or t if not. + If `t` is a matrix or higher order tensor, then the return value is + `t` reshaped to `[batch_size, beam_width] + s`. Otherwise `t` is + returned unchanged. Raises: - TypeError: If t is an instance of TensorArray. - ValueError: If the rank of t is not statically known. + TypeError: If `t` is an instance of `TensorArray`. + ValueError: If the rank of `t` is not statically known. """ _check_maybe(t) if t.shape.ndims >= 1: @@ -411,19 +412,19 @@ class BeamSearchDecoder(decoder.Decoder): def _maybe_merge_batch_beams(self, t, s): """Splits the tensor from a batch by beams into a batch of beams. - More exactly, t is a tensor of dimension [batch_size*beam_width, s]. We - reshape this into [batch_size, beam_width, s] + More exactly, `t` is a tensor of dimension `[batch_size * beam_width] + s`, + then we reshape it to `[batch_size, beam_width] + s`. Args: - t: Tensor of dimension [batch_size*beam_width, s] - s: Tensor, Python int, or TensorShape. + t: `Tensor` of dimension `[batch_size * beam_width] + s`. + s: `Tensor`, Python int, or `TensorShape`. Returns: - A reshaped version of t with dimension [batch_size, beam_width, s]. + A reshaped version of t with shape `[batch_size, beam_width] + s`. Raises: - TypeError: If t is an instance of TensorArray. - ValueError: If the rank of t is not statically known. + TypeError: If `t` is an instance of `TensorArray`. + ValueError: If the rank of `t` is not statically known. """ _check_maybe(t) if t.shape.ndims >= 2: @@ -521,14 +522,12 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size, # Calculate the continuation lengths by adding to all continuing beams. vocab_size = logits.shape[-1].value or array_ops.shape(logits)[-1] lengths_to_add = array_ops.one_hot( - indices=array_ops.tile( - array_ops.reshape(end_token, [1, 1]), [batch_size, beam_width]), + indices=array_ops.fill([batch_size, beam_width], end_token), depth=vocab_size, - on_value=constant_op.constant(0, dtype=dtypes.int64), - off_value=constant_op.constant(1, dtype=dtypes.int64), + on_value=np.int64(0), off_value=np.int64(1), dtype=dtypes.int64) - add_mask = (1 - math_ops.to_int64(previously_finished)) - lengths_to_add = array_ops.expand_dims(add_mask, 2) * lengths_to_add + add_mask = math_ops.to_int64(math_ops.logical_not(previously_finished)) + lengths_to_add *= array_ops.expand_dims(add_mask, 2) new_prediction_lengths = ( lengths_to_add + array_ops.expand_dims(prediction_lengths, 2)) @@ -592,9 +591,7 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size, # 1. Finished beams remain unchanged # 2. Beams that are now finished (EOS predicted) remain unchanged # 3. Beams that are not yet finished have their length increased by 1 - lengths_to_add = math_ops.to_int64( - math_ops.not_equal(next_word_ids, end_token)) - lengths_to_add = (1 - math_ops.to_int64(next_finished)) * lengths_to_add + lengths_to_add = math_ops.to_int64(math_ops.logical_not(next_finished)) next_prediction_len = _tensor_gather_helper( gather_indices=next_beam_ids, gather_from=beam_state.lengths, @@ -652,13 +649,20 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight): def _length_penalty(sequence_lengths, penalty_factor): """Calculates the length penalty. See https://arxiv.org/abs/1609.08144. + Returns the length penalty tensor: + ``` + [(5+sequence_lengths)/6]**penalty_factor + ``` + where all operations are performed element-wise. + Args: - sequence_lengths: The sequence length of all hypotheses, a tensor - of shape [beam_size, vocab_size]. + sequence_lengths: `Tensor`, the sequence lengths of each hypotheses. penalty_factor: A scalar that weights the length penalty. Returns: - The length penalty factor, a tensor fo shape [beam_size]. + If the penalty is `0`, returns the scalar `1.0`. Otherwise returns + the length penalty factor, a tensor with the same shape as + `sequence_lengths`. """ penalty_factor = ops.convert_to_tensor(penalty_factor, name="penalty_factor") penalty_factor.set_shape(()) # penalty should be a scalar. @@ -680,8 +684,7 @@ def _mask_probs(probs, eos_token, finished): eos_token: An int32 id corresponding to the EOS token to allocate probability to. finished: A boolean tensor of shape `[batch_size, beam_width]` that - specifies which - elements in the beam are finished already. + specifies which elements in the beam are finished already. Returns: A tensor of shape `[batch_size, beam_width, vocab_size]`, where unfinished @@ -689,10 +692,12 @@ def _mask_probs(probs, eos_token, finished): probability on the EOS token. """ vocab_size = array_ops.shape(probs)[2] - finished_mask = array_ops.expand_dims( - math_ops.to_float(1. - math_ops.to_float(finished)), 2) + finished_mask = math_ops.cast(array_ops.expand_dims(finished, 2), probs.dtype) + not_finished_mask = math_ops.cast( + array_ops.expand_dims(math_ops.logical_not(finished), 2), + probs.dtype) # These examples are not finished and we leave them - non_finished_examples = finished_mask * probs + non_finished_examples = not_finished_mask * probs # All finished examples are replaced with a vector that has all # probability on EOS finished_row = array_ops.one_hot( @@ -701,7 +706,7 @@ def _mask_probs(probs, eos_token, finished): dtype=probs.dtype, on_value=0., off_value=probs.dtype.min) - finished_examples = (1. - finished_mask) * finished_row + finished_examples = finished_mask * finished_row return finished_examples + non_finished_examples -- GitLab From b6d14a05cc51a6c32086cea8e6950ed45372fa7f Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Mon, 16 Oct 2017 11:37:02 -0700 Subject: [PATCH 026/573] Fix divergence between core.data and contrib.data Python tests. PiperOrigin-RevId: 172353443 --- .../kernel_tests/iterator_ops_cluster_test.py | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py index d7315a2526..45dfa13720 100644 --- a/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py +++ b/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py @@ -53,13 +53,8 @@ class IteratorClusterTest(test.TestCase): with self.assertRaises(errors.InvalidArgumentError): sess.run(get_next_op) - def testRemoteIteratorUsingRemoteCallOp(self): - worker_config = config_pb2.ConfigProto() - worker_config.device_count["CPU"] = 2 - worker, _ = test_util.create_local_cluster( - 1, 1, worker_config=worker_config) - - with ops.device("/job:worker/replica:0/task:0/cpu:1"): + def _testRemoteIteratorHelper(self, device0, device1, target): + with ops.device(device1): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) iterator_3 = dataset_3.make_one_shot_iterator() iterator_3_handle = iterator_3.string_handle() @@ -70,7 +65,7 @@ class IteratorClusterTest(test.TestCase): h, dataset_3.output_types, dataset_3.output_shapes) return remote_iterator.get_next() - with ops.device("/job:worker/replica:0/task:0/cpu:0"): + with ops.device(device0): target_placeholder = array_ops.placeholder(dtypes.string, shape=[]) remote_op = functional_ops.remote_call( args=[iterator_3_handle], @@ -78,32 +73,35 @@ class IteratorClusterTest(test.TestCase): f=_remote_fn, target=target_placeholder) - with session.Session(worker[0].target) as sess: - elem = sess.run( - remote_op, - feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"}) + with session.Session(target) as sess: + elem = sess.run(remote_op, feed_dict={target_placeholder: device1}) self.assertEqual(elem, [1]) # Fails when target is cpu:0 where the resource is not located. with self.assertRaises(errors.InvalidArgumentError): - sess.run( - remote_op, - feed_dict={ - target_placeholder: "/job:worker/replica:0/task:0/cpu:0" - }) - elem = sess.run( - remote_op, - feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"}) + sess.run(remote_op, feed_dict={target_placeholder: device0}) + elem = sess.run(iterator_3.get_next()) self.assertEqual(elem, [2]) - elem = sess.run( - remote_op, - feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"}) + elem = sess.run(remote_op, feed_dict={target_placeholder: device1}) self.assertEqual(elem, [3]) with self.assertRaises(errors.OutOfRangeError): - sess.run( - remote_op, - feed_dict={ - target_placeholder: "/job:worker/replica:0/task:0/cpu:1" - }) + sess.run(remote_op, feed_dict={target_placeholder: device1}) + + def testRemoteIteratorUsingRemoteCallOp(self): + worker_config = config_pb2.ConfigProto() + worker_config.device_count["CPU"] = 2 + worker, _ = test_util.create_local_cluster( + 1, 1, worker_config=worker_config) + + self._testRemoteIteratorHelper("/job:worker/replica:0/task:0/cpu:0", + "/job:worker/replica:0/task:0/cpu:1", + worker[0].target) + + def testRemoteIteratorUsingRemoteCallOpCrossProcess(self): + workers, _ = test_util.create_local_cluster(2, 1) + + self._testRemoteIteratorHelper("/job:worker/replica:0/task:0/cpu:0", + "/job:worker/replica:0/task:1/cpu:0", + workers[0].target) if __name__ == "__main__": -- GitLab From 2ed69577b08e5e8845619748249ecd41dc0f7c87 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Oct 2017 12:37:40 -0700 Subject: [PATCH 027/573] Add fast_tensor_util.cpp to .gitignore (#13749) While working on building TensorFlow I noticed that a file `fast_tensor_util.cpp` is generated: ```sh ubuntu@ubuntu:~/tensorflow$ git status On branch master Your branch is up-to-date with 'origin/master'. Untracked files: (use "git add ..." to include in what will be committed) tensorflow/python/framework/fast_tensor_util.cpp nothing added to commit but untracked files present (use "git add" to track) ubuntu@ubuntu:~/tensorflow$ ``` This fix adds `fast_tensor_util.cpp` to .gitignore so that it will not be added inadvertently when adding commit with `git add -A`. Signed-off-by: Yong Tang --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 09734fe497..9572a3e97c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ cmake_build/ .idea/** /build/ /tensorflow/core/util/version_info.cc +/tensorflow/python/framework/fast_tensor_util.cpp -- GitLab From 8de821fc169fb9bad8be681801e8551171f8e44a Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 16 Oct 2017 12:42:35 -0700 Subject: [PATCH 028/573] make_vjp in eager PiperOrigin-RevId: 172363016 --- tensorflow/python/eager/backprop.py | 56 ++++++++++++++++++++++++ tensorflow/python/eager/backprop_test.py | 10 +++++ 2 files changed, 66 insertions(+) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 0060dd0c1c..7f1a770513 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -581,6 +581,62 @@ def val_and_grad_function(f, params=None): return decorated +def make_vjp(f, params=None): + """Returns a function that computes f and is vjp w.r.t. params. + + The term "vjp" here is an abbreviation for vector-jacobian product. + + Args: + f: the function to be differentiated. + params: the parameters (numbers or names) to differentiate with respect to. + A value of None will differentiate with respect to all parameters. + + Returns: + A function, which when called, returns a tuple (value, vjp), where: + - value is the result of calling f. + - vjp is a function, which takes a vector as an argument and + returns the product of that vector with the Jacobian of f. + Providing no argument to vjp is equivalent to providing a + vector of ones. + + For example, + ```python + def f(x): + return x * x + + wrapped_fn = tfe.make_vjp(f) + result, vjp = wrapped_fn(tf.constant(3.0)) + # result is 9.0 + vjp() # the vjp function rturns 6.0 + + """ + + parameter_positions = _get_arg_spec(f, params) + + def decorated(*args, **kwds): + """Computes the value and gradient of the decorated function.""" + assert not kwds, "The gradient function can't take keyword arguments." + tape.push_new_tape() + sources = [] + args = [ + ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] + for i in range(len(args)) + ] + args = _ensure_unique_tensor_objects(parameter_positions, args) + for i in parameter_positions: + sources.append(args[i]) + tape.watch(args[i]) + result = f(*args) + t = tape.pop_tape() + def vjp(dy=None): + return imperative_grad.imperative_grad( + _default_vspace, t, nest.flatten(result), sources, + output_gradients=nest.flatten(dy) if dy is not None else None) + return result, vjp + + return decorated + + def _aggregate_grads(gradients): """Aggregate gradients from multiple sources. diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index d53c69afcc..9083e3a712 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -168,6 +168,16 @@ class BackpropTest(test.TestCase): grad = backprop.gradients_function(second, [0])(f)[0] self.assertAllEqual([[0.0]], grad.numpy()) + def testMakeVJP(self): + + def f(x): + return x * x + + wrapped_fn = backprop.make_vjp(f) + result, vjp = wrapped_fn(constant_op.constant(3.0)) + self.assertEqual(result.numpy(), 9.0) + self.assertEqual(vjp(2.0)[0].numpy(), 12.0) + def testGradGrad(self): def sq(x): -- GitLab From d4efe4dd39d6894779bb09462b2af8161b3dedad Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Mon, 16 Oct 2017 12:45:10 -0700 Subject: [PATCH 029/573] Implement set_shape for EagerTensors for compatibiity with ops that call it Checks if shape is not compatible with the Eager tensor's shape, raises an error if it is not. PiperOrigin-RevId: 172363347 --- tensorflow/python/eager/BUILD | 1 + tensorflow/python/eager/ops_test.py | 11 +++++++++++ tensorflow/python/framework/ops.py | 9 ++++++--- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 69b96df87c..5a2592287c 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -408,6 +408,7 @@ py_test( "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", "//tensorflow/python:sparse_ops", + "//tensorflow/python:tensor_shape", "//third_party/py/numpy", ], ) diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 7d54b8d2d8..78423468ea 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.layers import core from tensorflow.python.ops import array_ops @@ -302,6 +303,16 @@ class OpsTest(test_util.TensorFlowTestCase): def testIdentity(self): self.assertEqual(2, array_ops.identity(2).numpy()) + def testIncompatibleSetShape(self): + x = constant_op.constant(1) + with self.assertRaises(ValueError): + x.set_shape((1, 2)) + + def testCompatibleSetShape(self): + x = constant_op.constant([[1, 2]]) + x.set_shape(tensor_shape.TensorShape([None, 2])) + self.assertEqual(x.get_shape(), (1, 2)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 6077d602c4..a52a0cfc2d 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -727,6 +727,12 @@ class _EagerTensorBase(Tensor): def __nonzero__(self): return self.__bool__() + def set_shape(self, shape): + if not self.shape.is_compatible_with(shape): + raise ValueError( + "EagerTensor's shape %s is not compatible with supplied shape %s" % + (self.shape, shape)) + # Methods not supported / implemented for Eager Tensors. @property def op(self): @@ -740,9 +746,6 @@ class _EagerTensorBase(Tensor): def name(self): raise NotImplementedError("name not supported for Eager Tensors.") - def set_shape(self, shape): - raise NotImplementedError("set_shape not supported for Eager Tensors.") - @property def value_index(self): raise NotImplementedError("value_index not supported for Eager Tensors.") -- GitLab From 58eee58840094782b22dbb8981b513df3797eac0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 13:03:47 -0700 Subject: [PATCH 030/573] Remove broken link. PiperOrigin-RevId: 172366027 --- tensorflow/contrib/gan/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md index 10458a2458..5d74df3ef7 100644 --- a/tensorflow/contrib/gan/README.md +++ b/tensorflow/contrib/gan/README.md @@ -51,9 +51,10 @@ network to evaluate your unconditional generative model. You can also also use your own pretrained classifier for more specific performance numbers, or use other methods for evaluating conditional generative models. -* [examples](https://github.com/tensorflow/models/tree/master/gan/): +* examples (coming soon): See examples of how to use TFGAN to make GAN training easier, or use the more complicated examples to jumpstart your -own project. +own project. These include unconditional and conditional GANs, InfoGANs, +adversarial losses on existing networks, and image-to-image translation. ## Training a GAN model -- GitLab From b1128a402d473cc6a43c99a081446c1b45305dd9 Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Mon, 16 Oct 2017 13:10:20 -0700 Subject: [PATCH 031/573] Move global_step_read dependency to model_fn instead of input_fn. PiperOrigin-RevId: 172366972 --- tensorflow/python/estimator/estimator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 4dfc53aadf..00a57f11dc 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -708,11 +708,11 @@ class Estimator(object): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access + features, labels = self._get_features_and_labels_from_input_fn( + input_fn, model_fn_lib.ModeKeys.TRAIN) with ops.control_dependencies([global_step_read_tensor]): - features, labels = self._get_features_and_labels_from_input_fn( - input_fn, model_fn_lib.ModeKeys.TRAIN) - estimator_spec = self._call_model_fn( - features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) + estimator_spec = self._call_model_fn( + features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) # Check if the user created a loss summary, and add one if they didn't. # We assume here that the summary is called 'loss'. If it is not, we will # make another one with the name 'loss' to ensure it shows up in the right -- GitLab From 940455b04c843333f1a359fcb2412ebba1780a7f Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Mon, 16 Oct 2017 13:33:55 -0700 Subject: [PATCH 032/573] Creating a patch for the wrong links that still point to dev. (#13752) --- tensorflow/docs_src/install/install_linux.md | 22 +++++++++---------- tensorflow/docs_src/install/install_mac.md | 10 ++++----- .../docs_src/install/install_sources.md | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 2b488cc4f5..9d204cc246 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -184,7 +184,7 @@ Take the following steps to install TensorFlow with Virtualenv: virtualenv environment: 
(tensorflow)$ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl If you encounter installation problems, see [Common Installation Problems](#common_installation_problems). @@ -289,7 +289,7 @@ take the following steps: 
      $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
If this step fails, see @@ -476,7 +476,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl @@ -644,14 +644,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -663,14 +663,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -682,14 +682,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
@@ -701,14 +701,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index efd977089b..6da22784bf 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -109,7 +109,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -230,7 +230,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -339,7 +339,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl @@ -512,7 +512,7 @@ This section documents the relevant values for Mac OS installations. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
@@ -520,7 +520,7 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 6114496cd5..b853d87816 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -349,10 +349,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.4.0dev on Linux: +for TensorFlow 1.4.0rc0 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0dev-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc0-py2-none-any.whl
## Validate your installation -- GitLab From 24f9c6e0dbd449624aa1db543550ec412975492e Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Mon, 16 Oct 2017 13:40:08 -0700 Subject: [PATCH 033/573] Add support for saving DT_VARIANT tensors in TensorBundle. Add support for reading Varint64 to InputBuffer. PiperOrigin-RevId: 172371104 --- tensorflow/core/lib/core/coding.h | 3 + tensorflow/core/lib/io/inputbuffer.cc | 26 +++- tensorflow/core/lib/io/inputbuffer.h | 26 ++++ tensorflow/core/lib/io/inputbuffer_test.cc | 39 ++++++ tensorflow/core/util/tensor_bundle/BUILD | 1 + .../core/util/tensor_bundle/tensor_bundle.cc | 128 +++++++++++++++++- .../util/tensor_bundle/tensor_bundle_test.cc | 75 ++++++++++ 7 files changed, 289 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/lib/core/coding.h b/tensorflow/core/lib/core/coding.h index 77d52a909b..8265aec870 100644 --- a/tensorflow/core/lib/core/coding.h +++ b/tensorflow/core/lib/core/coding.h @@ -31,6 +31,9 @@ namespace core { // Maximum number of bytes occupied by a varint32. static const int kMaxVarint32Bytes = 5; +// Maximum number of bytes occupied by a varint64. +static const int kMaxVarint64Bytes = 10; + // Lower-level versions of Put... that write directly into a character buffer // REQUIRES: dst has enough space for the value being written extern void EncodeFixed16(char* dst, uint16 value); diff --git a/tensorflow/core/lib/io/inputbuffer.cc b/tensorflow/core/lib/io/inputbuffer.cc index 7efe2dc543..4d35af49b2 100644 --- a/tensorflow/core/lib/io/inputbuffer.cc +++ b/tensorflow/core/lib/io/inputbuffer.cc @@ -116,17 +116,35 @@ Status InputBuffer::ReadNBytes(int64 bytes_to_read, char* result, } Status InputBuffer::ReadVarint32Fallback(uint32* result) { + Status s = ReadVarintFallback(result, core::kMaxVarint32Bytes); + if (errors::IsDataLoss(s)) { + return errors::DataLoss("Stored data is too large to be a varint32."); + } + return s; +} + +Status InputBuffer::ReadVarint64Fallback(uint64* result) { + Status s = ReadVarintFallback(result, core::kMaxVarint64Bytes); + if (errors::IsDataLoss(s)) { + return errors::DataLoss("Stored data is too large to be a varint64."); + } + return s; +} + +template +Status InputBuffer::ReadVarintFallback(T* result, int max_bytes) { uint8 scratch = 0; - char* p = reinterpret_cast(&scratch); + auto* p = reinterpret_cast(&scratch); size_t unused_bytes_read = 0; *result = 0; - for (int shift = 0; shift <= 28; shift += 7) { + for (int index = 0; index < max_bytes; index++) { + int shift = 7 * index; TF_RETURN_IF_ERROR(ReadNBytes(1, p, &unused_bytes_read)); - *result |= (scratch & 127) << shift; + *result |= (static_cast(scratch) & 127) << shift; if (!(scratch & 128)) return Status::OK(); } - return errors::DataLoss("Stored data is too large to be a varint32."); + return errors::DataLoss("Stored data longer than ", max_bytes, " bytes."); } Status InputBuffer::SkipNBytes(int64 bytes_to_skip) { diff --git a/tensorflow/core/lib/io/inputbuffer.h b/tensorflow/core/lib/io/inputbuffer.h index 94a8cfd39b..b3740f396c 100644 --- a/tensorflow/core/lib/io/inputbuffer.h +++ b/tensorflow/core/lib/io/inputbuffer.h @@ -60,6 +60,9 @@ class InputBuffer { // Reads a single varint32. Status ReadVarint32(uint32* result); + // Reads a single varint64. + Status ReadVarint64(uint64* result); + // Like ReadNBytes() without returning the bytes read. Status SkipNBytes(int64 bytes_to_skip); @@ -82,6 +85,15 @@ class InputBuffer { // Internal slow-path routine used by ReadVarint32(). Status ReadVarint32Fallback(uint32* result); + // Internal slow-path routine used by ReadVarint64(). + Status ReadVarint64Fallback(uint64* result); + + // Helper method for reading a varint which can span at max `max_bytes`. + // If the varint is longer, a DataLoss error status is returned. + // If end of file is reached while reading, OutOfRange error is returned. + template + Status ReadVarintFallback(T* result, int max_bytes); + RandomAccessFile* file_; // Not owned int64 file_pos_; // Next position to read from in "file_" size_t size_; // Size of "buf_" @@ -109,6 +121,20 @@ inline Status InputBuffer::ReadVarint32(uint32* result) { } } +// Inlined for performance. +inline Status InputBuffer::ReadVarint64(uint64* result) { + if (pos_ + core::kMaxVarint64Bytes <= limit_) { + // Fast path: directly parse from buffered data. + // Reads strictly from the range [pos_, limit_). + const char* offset = core::GetVarint64Ptr(pos_, limit_, result); + if (offset == nullptr) return errors::OutOfRange("Parsed past limit."); + pos_ = const_cast(offset); + return Status::OK(); + } else { + return ReadVarint64Fallback(result); + } +} + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/inputbuffer_test.cc b/tensorflow/core/lib/io/inputbuffer_test.cc index 6771697a16..6be1f819c2 100644 --- a/tensorflow/core/lib/io/inputbuffer_test.cc +++ b/tensorflow/core/lib/io/inputbuffer_test.cc @@ -329,5 +329,44 @@ TEST(InputBuffer, ReadVarint32) { } } +TEST(InputBuffer, ReadVarint64) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/inputbuffer_test"; + + // Generates data. + std::vector data; + uint64 i = 0; + for (; i < (1U << 10); i += 1) data.push_back(i); + for (; i < (1U << 15); i += 5) data.push_back(i); + for (; i < (1U << 31); i += 164817) data.push_back(i); + for (; i < (1ULL << 63); i += 16481797854795663UL) data.push_back(i); + data.push_back(std::numeric_limits::max()); + + // Writes the varints. + { + std::unique_ptr file; + TF_CHECK_OK(env->NewWritableFile(fname, &file)); + string varint; + for (uint64 number : data) { + varint.clear(); + core::PutVarint64(&varint, number); + TF_CHECK_OK(file->Append(StringPiece(varint))); + } + } + + for (auto buf_size : BufferSizes()) { + std::unique_ptr file; + TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); + io::InputBuffer in(file.get(), buf_size); + uint64 result = 0; + + for (uint64 expected : data) { + TF_ASSERT_OK(in.ReadVarint64(&result)); + EXPECT_EQ(expected, result); + } + EXPECT_TRUE(errors::IsOutOfRange(in.ReadVarint64(&result))); + } +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/util/tensor_bundle/BUILD b/tensorflow/core/util/tensor_bundle/BUILD index 4e957ec3df..166bd0f659 100644 --- a/tensorflow/core/util/tensor_bundle/BUILD +++ b/tensorflow/core/util/tensor_bundle/BUILD @@ -66,6 +66,7 @@ tf_cc_test( srcs = ["tensor_bundle_test.cc"], deps = [ ":tensor_bundle", + "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc index 02eb042a0b..d0e54b7e47 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc @@ -22,10 +22,14 @@ limitations under the License. #include #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/framework/tensor_shape.pb_text.h" #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb_text.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_op_registry.h" +#include "tensorflow/core/framework/variant_tensor_data.h" #include "tensorflow/core/framework/versions.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/lib/core/coding.h" @@ -109,6 +113,64 @@ Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements, return Status::OK(); } +Status ReadVariantTensor(io::InputBuffer* buffered_file, Tensor* ret, + size_t offset, size_t size, uint32* actual_crc32c) { + // On-disk format: + // [varint64 len1][bytes variant1][4 byte checksum] + // .. + // [varint64 lenN][bytes variantN][4 byte checksum] + // Var "crc32c" checksums all the lens, variant bytes, individual variant + // checksums (as uint32, not varint32 bytes). + if (size == 0) return Status::OK(); + size_t num_elements = ret->NumElements(); + + // Reads the actual string bytes. + TF_RETURN_IF_ERROR(buffered_file->Seek(offset)); + for (size_t i = 0; i < num_elements; ++i) { + // Read the serialized variant length. + uint64 string_length = 0; + TF_RETURN_IF_ERROR(buffered_file->ReadVarint64(&string_length)); + *actual_crc32c = crc32c::Extend( + *actual_crc32c, reinterpret_cast(&string_length), + sizeof(uint64)); + // Read the actual serialized variant. + string buffer; + buffer.resize(string_length); + size_t bytes_read = 0; + TF_RETURN_IF_ERROR( + buffered_file->ReadNBytes(string_length, &buffer[0], &bytes_read)); + *actual_crc32c = crc32c::Extend(*actual_crc32c, buffer.data(), bytes_read); + VariantTensorDataProto proto; + proto.ParseFromString(buffer); + Variant v = proto; + if (!DecodeUnaryVariant(&v)) { + return errors::Internal("Could not decode variant with type_name: \"", + v.TypeName(), "\". Perhaps you forgot to ", + "register a decoder via ", + "REGISTER_UNARY_VARIANT_DECODE_FUNCTION?"); + } + + // Read the checksum. + uint32 checksum = 0; + size_t unused_bytes_read = 0; + TF_RETURN_IF_ERROR(buffered_file->ReadNBytes( + sizeof(uint32), reinterpret_cast(&checksum), + &unused_bytes_read)); + if (crc32c::Unmask(checksum) != *actual_crc32c) { + return errors::DataLoss( + "The checksum after Variant ", i, " does not match.", + " Expected: ", strings::Printf("%08u", crc32c::Unmask(checksum)), + " Actual: ", strings::Printf("%08u", *actual_crc32c)); + } + *actual_crc32c = crc32c::Extend( + *actual_crc32c, reinterpret_cast(&checksum), sizeof(uint32)); + + ret->flat()(i) = std::move(v); + } + + return Status::OK(); +} + char* GetBackingBuffer(const Tensor& val) { CHECK(DataTypeCanUseMemcpy(val.dtype())) << val.dtype(); return const_cast(val.tensor_data().data()); @@ -134,6 +196,7 @@ Status ParseEntryProto(StringPiece key, StringPiece value, Status WriteTensor(const Tensor& val, FileOutputBuffer* out, size_t* bytes_written) { DCHECK_NE(val.dtype(), DT_STRING); + DCHECK_NE(val.dtype(), DT_VARIANT); *bytes_written = val.TotalBytes(); char* buf = GetBackingBuffer(val); VLOG(1) << "Appending " << *bytes_written << " bytes to file"; @@ -188,6 +251,54 @@ Status WriteStringTensor(const Tensor& val, FileOutputBuffer* out, return Status::OK(); } +Status WriteVariantTensor(const Tensor& val, FileOutputBuffer* out, + size_t* bytes_written, uint32* crc32c) { + // On-disk format: + // [varint64 len1][bytes variant1][4 byte checksum] + // .. + // [varint64 lenN][bytes variantN][4 byte checksum] + // Var "crc32c" checksums all the lens, variant bytes, individual variant + // checksums (as uint32, not varint32 bytes). + DCHECK_EQ(val.dtype(), DT_VARIANT); + + *crc32c = 0; + *bytes_written = 0; + for (int64 i = 0; i < val.NumElements(); ++i) { + VariantTensorData data; + val.flat()(i).Encode(&data); + VariantTensorDataProto proto; + data.ToProto(&proto); + string elem; + proto.SerializeToString(&elem); + + // Write the length of the serialized variant. + DCHECK_EQ(elem.size(), static_cast(elem.size())); + const auto elem_size = static_cast(elem.size()); + string len; + core::PutVarint64(&len, elem_size); + TF_RETURN_IF_ERROR(out->Append(len)); + *crc32c = crc32c::Extend(*crc32c, reinterpret_cast(&elem_size), + sizeof(uint64)); + *bytes_written += sizeof(uint64); + + // Write the serialized variant. + TF_RETURN_IF_ERROR(out->Append(elem)); + *crc32c = crc32c::Extend(*crc32c, elem.data(), elem.size()); + *bytes_written += elem.size(); + + // Write the checksum. + const uint32 length_checksum = crc32c::Mask(*crc32c); + TF_RETURN_IF_ERROR(out->Append(StringPiece( + reinterpret_cast(&length_checksum), sizeof(uint32)))); + *crc32c = + crc32c::Extend(*crc32c, reinterpret_cast(&length_checksum), + sizeof(uint32)); + *bytes_written += sizeof(uint32); + } + + return Status::OK(); +} + // Reads file[offset:offset+size) into destination[0:size). Each Read() copies // at most "buffer_size" bytes. // @@ -312,11 +423,13 @@ Status BundleWriter::Add(StringPiece key, const Tensor& val) { size_t data_bytes_written = 0; uint32 crc32c = 0; out_->clear_crc32c(); - if (val.dtype() != DT_STRING) { + if (val.dtype() == DT_STRING) { + status_ = WriteStringTensor(val, out_.get(), &data_bytes_written, &crc32c); + } else if (val.dtype() == DT_VARIANT) { + status_ = WriteVariantTensor(val, out_.get(), &data_bytes_written, &crc32c); + } else { status_ = WriteTensor(val, out_.get(), &data_bytes_written); crc32c = out_->crc32c(); - } else { - status_ = WriteStringTensor(val, out_.get(), &data_bytes_written, &crc32c); } if (status_.ok()) { @@ -707,13 +820,13 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { } // Validates the "size" field. - if (entry.dtype() != DT_STRING) { + if (entry.dtype() != DT_STRING && entry.dtype() != DT_VARIANT) { if (entry.size() != ret->TotalBytes()) { return errors::DataLoss("Invalid size in bundle entry: key ", key(), "; stored size ", entry.size(), "; expected size ", ret->TotalBytes()); } - } else { + } else if (entry.dtype() == DT_STRING) { // Relaxes the check for string tensors as follows: // entry.size() == bytes(varint lengths) + bytes(data) // >= NumElems + bytes(data), since size bytes(varint) >= 1. @@ -752,6 +865,11 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { entry.size(), 8 << 20 /* 8MB buffer */, backing_buffer)); actual_crc32c = crc32c::Value(backing_buffer, entry.size()); + } else if (entry.dtype() == DT_VARIANT) { + // Relies on io::InputBuffer's buffering, because we issue many neighboring + // reads for a single string tensor. + TF_RETURN_IF_ERROR(ReadVariantTensor(buffered_file, ret, entry.offset(), + entry.size(), &actual_crc32c)); } else { // Relies on io::InputBuffer's buffering, because we issue many neighboring // reads for a single string tensor. diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc index 4ee1578510..341aae36f4 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" @@ -64,6 +66,30 @@ void Expect(BundleReader* reader, const string& key, test::ExpectTensorEqual(val, expected_val); } +template +void ExpectVariant(BundleReader* reader, const string& key, + const Tensor& expected_t) { + // Tests for Contains(). + EXPECT_TRUE(reader->Contains(key)); + // Tests for LookupDtypeAndShape(). + DataType dtype; + TensorShape shape; + TF_ASSERT_OK(reader->LookupDtypeAndShape(key, &dtype, &shape)); + // Tests for Lookup(), checking tensor contents. + EXPECT_EQ(expected_t.dtype(), dtype); + EXPECT_EQ(expected_t.shape(), shape); + Tensor actual_t(dtype, shape); + TF_ASSERT_OK(reader->Lookup(key, &actual_t)); + for (int i = 0; i < expected_t.NumElements(); i++) { + Variant actual_var = actual_t.flat()(i); + Variant expected_var = expected_t.flat()(i); + EXPECT_EQ(actual_var.TypeName(), expected_var.TypeName()); + auto* actual_val = actual_var.get(); + auto* expected_val = expected_var.get(); + EXPECT_EQ(*expected_val, *actual_val); + } +} + template void ExpectNext(BundleReader* reader, const Tensor& expected_val) { EXPECT_TRUE(reader->Valid()); @@ -460,6 +486,55 @@ TEST(TensorBundleTest, StringTensors) { } } +class VariantObject { + public: + VariantObject() {} + VariantObject(const string& metadata, int64 value) + : metadata_(metadata), value_(value) {} + + string TypeName() const { return "TEST VariantObject"; } + void Encode(VariantTensorData* data) const { + data->set_type_name(TypeName()); + data->set_metadata(metadata_); + Tensor val_t = Tensor(DT_INT64, TensorShape({})); + val_t.scalar()() = value_; + *(data->add_tensors()) = val_t; + } + bool Decode(const VariantTensorData& data) { + EXPECT_EQ(data.type_name(), TypeName()); + data.get_metadata(&metadata_); + EXPECT_EQ(data.tensors_size(), 1); + value_ = data.tensors(0).scalar()(); + return true; + } + bool operator==(const VariantObject other) const { + return metadata_ == other.metadata_ && value_ == other.value_; + } + string metadata_; + int64 value_; +}; + +REGISTER_UNARY_VARIANT_DECODE_FUNCTION(VariantObject, "TEST VariantObject"); + +TEST(TensorBundleTest, VariantTensors) { + { + BundleWriter writer(Env::Default(), Prefix("foo")); + TF_EXPECT_OK( + writer.Add("variant_tensor", + test::AsTensor({VariantObject("test", 10), + VariantObject("test1", 20)}))); + TF_ASSERT_OK(writer.Finish()); + } + { + BundleReader reader(Env::Default(), Prefix("foo")); + TF_ASSERT_OK(reader.status()); + ExpectVariant( + &reader, "variant_tensor", + test::AsTensor( + {VariantObject("test", 10), VariantObject("test1", 20)})); + } +} + TEST(TensorBundleTest, DirectoryStructure) { Env* env = Env::Default(); // Writes two bundles. -- GitLab From b46b741b7a8f04049b451be5299c61a373ec7612 Mon Sep 17 00:00:00 2001 From: Russell Power Date: Mon, 16 Oct 2017 13:46:18 -0700 Subject: [PATCH 034/573] Close session on infeed error. This should fix most of the cases where the client process hangs waiting for the main training loop to exit. PiperOrigin-RevId: 172371951 --- .../contrib/tpu/python/tpu/tpu_estimator.py | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index de6c8140c6..04e0719a1b 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -312,17 +312,25 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController): def _input_thread_fn_for_loading(self, session, enqueue_ops): count = 0 - while True: - signal = self._signal_queue.get() - if signal == _SIGNAL.STOP: - logging.info('Stop Infeed input thread.') - return - - iterations = signal - for i in range(iterations): - logging.debug('Infeed enqueue for iteration (%d, %d)', count, i) - session.run(enqueue_ops) - count += 1 + try: + while True: + signal = self._signal_queue.get() + if signal == _SIGNAL.STOP: + logging.info('Stop Infeed input thread.') + return + + iterations = signal + for i in range(iterations): + logging.debug('Infeed enqueue for iteration (%d, %d)', count, i) + session.run(enqueue_ops) + count += 1 + except Exception: # pylint: disable=broad-except + logging.error( + 'Failed running infeed, closing session.\n' + 'You may see an exception from your main session after this.', + exc_info=1 + ) + session.close() def join(self): logging.info('Waiting for Infeed Thread to exit.') -- GitLab From a36ff5499df443d768fd2f4ff810f9daba30d35a Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 16 Oct 2017 13:50:36 -0700 Subject: [PATCH 035/573] Respect __array__ and __array_interface__ for string types __array__ fixes use-cases like: import tensorflow as tf import pandas as pd series = pd.Series(['a','b','c']) tf.constant(series) df = pd.DataFrame({'a':[1,2,3],'b':['a','b','c']}) tf.data.Dataset.from_tensor_slices(dict(df)) PiperOrigin-RevId: 172372593 --- tensorflow/python/framework/tensor_util.py | 11 ++++-- .../python/framework/tensor_util_test.py | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py index 414c61e930..63324e5977 100644 --- a/tensorflow/python/framework/tensor_util.py +++ b/tensorflow/python/framework/tensor_util.py @@ -362,10 +362,15 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): nparray = values.astype(dtype.as_numpy_dtype) else: nparray = values - elif callable(getattr(values, "__array__", None)): - # If a class has the __array__ method, then it is possible to convert - # to numpy array. + elif callable(getattr(values, "__array__", None)) or isinstance( + getattr(values, "__array_interface__", None), dict): + # If a class has the __array__ method, or __array_interface__ dict, then it + # is possible to convert to numpy array. nparray = np.asarray(values, dtype=dtype) + + # This is the preferred way to create an array from the object, so replace + # the `values` with the array so that _FlattenToStrings is not run. + values = nparray else: if values is None: raise ValueError("None values not supported.") diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py index c4937de936..dda72fc0c8 100644 --- a/tensorflow/python/framework/tensor_util_test.py +++ b/tensorflow/python/framework/tensor_util_test.py @@ -492,6 +492,45 @@ class TensorUtilTest(test.TestCase): self.assertEquals(np.object, a.dtype) self.assertAllEqual(np.array([[b"a", b"ab"], [b"abc", b"abcd"]]), a) + def testArrayMethod(self): + + class Wrapper(object): + + def __array__(self): + return np.array([b"foo", b"bar", b"baz"]) + + t = tensor_util.make_tensor_proto(Wrapper(), shape=[1, 3]) + self.assertProtoEquals(""" + dtype: DT_STRING + tensor_shape { dim { size: 1 } dim { size: 3 } } + string_val: "foo" + string_val: "bar" + string_val: "baz" + """, t) + a = tensor_util.MakeNdarray(t) + self.assertEquals(np.object, a.dtype) + self.assertAllEqual(np.array([[b"foo", b"bar", b"baz"]]), a) + + def testArrayInterface(self): + + class Wrapper(object): + + @property + def __array_interface__(self): + return np.array([b"foo", b"bar", b"baz"]).__array_interface__ + + t = tensor_util.make_tensor_proto(Wrapper(), shape=[1, 3]) + self.assertProtoEquals(""" + dtype: DT_STRING + tensor_shape { dim { size: 1 } dim { size: 3 } } + string_val: "foo" + string_val: "bar" + string_val: "baz" + """, t) + a = tensor_util.MakeNdarray(t) + self.assertEquals(np.object, a.dtype) + self.assertAllEqual(np.array([[b"foo", b"bar", b"baz"]]), a) + def testStringTuple(self): t = tensor_util.make_tensor_proto((b"a", b"ab", b"abc", b"abcd")) self.assertProtoEquals(""" -- GitLab From 36ff23d81b271aaf4a4e106c042f76e80484d769 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 14:01:43 -0700 Subject: [PATCH 036/573] Batch norm folding immediately fails if FusedBatchNorm ops are present. PiperOrigin-RevId: 172374244 --- .../quantize/python/fold_batch_norms.py | 4 +++ .../quantize/python/fold_batch_norms_test.py | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py index c9d16fb329..c416689510 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py @@ -40,6 +40,10 @@ def FoldBatchNorms(graph): Raises: ValueError: When batch norm folding fails. """ + # Fail immediately when the graph contains unsupported fused batch norm ops. + if any(op for op in graph.get_operations() if op.type == 'FusedBatchNorm'): + raise ValueError('Fused batch norm is not supported') + input_to_ops_map = input_to_ops.InputToOps(graph) for bn in common.BatchNormGroups(graph): diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py index 4f11188a55..ddedb0a2c0 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py @@ -57,6 +57,34 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): for parameters in parameters_list: test_fn(parameters[0], parameters[1], parameters[2]) + def testFailsWithFusedBatchNorm(self): + self._RunTestOverParameters(self._TestFailsWithFusedBatchNorm) + + def _TestFailsWithFusedBatchNorm(self, relu, relu_op_name, with_bypass): + """Tests that batch norm fails when fused batch norm ops are present.""" + g = ops.Graph() + with g.as_default(): + batch_size, height, width = 5, 128, 128 + inputs = array_ops.zeros((batch_size, height, width, 3)) + out_depth = 3 if with_bypass else 32 + stride = 1 if with_bypass else 2 + activation_fn = None if with_bypass else relu + batch_norm_params = _DEFAULT_BATCH_NORM_PARAMS.copy() + batch_norm_params['fused'] = True + scope = 'test/test2' if with_bypass else 'test' + node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=batch_norm_params, + scope=scope) + if with_bypass: + node = math_ops.add(inputs, node, name='test/Add') + relu(node, name='test/' + relu_op_name) + + with self.assertRaises(ValueError): + fold_batch_norms.FoldBatchNorms(g) + def _TestFoldConv2d(self, relu, relu_op_name, with_bypass): """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. -- GitLab From 7b3ea8e6176319467cb1a49a1a662d868a205b91 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Mon, 16 Oct 2017 14:18:10 -0700 Subject: [PATCH 037/573] [TF2XLA] Expand comparator and use consistently in sorting arguments. PiperOrigin-RevId: 172376836 --- .../tf2xla/functionalize_control_flow.cc | 32 ++++++++++--------- .../tf2xla/functionalize_control_flow_test.cc | 20 ++++++------ 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc index abfc856904..35b6960a98 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc @@ -590,8 +590,13 @@ class FunctionalizeCond { // id in the original graph. struct CondArgs { struct CondCmp { - bool operator()(const Node* a, const Node* b) const { - return a->id() < b->id(); + bool operator()(const Node* lhs, const Node* rhs) const { + bool lhs_is_resource = + lhs->num_inputs() > 0 ? (lhs->input_type(0) == DT_RESOURCE) : false; + bool rhs_is_resource = + rhs->num_inputs() > 0 ? (rhs->input_type(0) == DT_RESOURCE) : false; + return std::tie(lhs_is_resource, lhs->name()) < + std::tie(rhs_is_resource, rhs->name()); } }; Node* conditional = nullptr; @@ -710,7 +715,7 @@ std::ostream& operator<<(std::ostream& os, // between the nodes and the nodes in each cluster. string DebugString(const Graph& graph, FunctionalizeCond::ClusterHandle::Vector* clusters) { - string ret = "digraph {\ncompound=true;labeljust=\"r\";\n"; + string ret = "digraph {\ncompound=true;labeljust=\"r\";ranksep=0.24\n"; std::map subgraphs; for (Node* n : graph.nodes()) { if (n->IsOp()) { @@ -720,8 +725,8 @@ string DebugString(const Graph& graph, } for (auto kv : subgraphs) { strings::StrAppend(&ret, "subgraph cluster_", kv.first.ToString(), " {\n", - "label = \"", kv.first.ToString(), "\";\n", kv.second, - "}\n"); + "style=filled; color=lightgrey;", "label = \"", + kv.first.ToString(), "\";\n", kv.second, "}\n"); } for (Node* n : graph.nodes()) { if (!n->IsOp()) { @@ -1110,11 +1115,6 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args, DataType dtype = arg->input_type(0); TF_ASSIGN_OR_RETURN(Node * arg_node, BuildArgNode(body, dtype, arg_count++)); - if (dtype == DT_RESOURCE) { - bool constant; - TF_RETURN_IF_ERROR(GetNodeAttr(arg->attrs(), "is_constant", &constant)); - TF_RET_CHECK(constant); - } node_map.at(arg->id()) = arg_node; squash_src_outputs.at(arg->id()) = true; } @@ -1247,9 +1247,7 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { // Sort the outputs by ID to produce more stable output. std::vector outputs(merge_cluster->merge_nodes.begin(), merge_cluster->merge_nodes.end()); - std::sort( - outputs.begin(), outputs.end(), - [](const Node* lhs, const Node* rhs) { return lhs->id() < rhs->id(); }); + std::sort(outputs.begin(), outputs.end(), CondArgs::CondCmp()); // Extract bodies and builds a If operator. TF_ASSIGN_OR_RETURN(Node * if_node, @@ -1370,7 +1368,7 @@ Status FunctionalizeCond::Functionalize(Graph* graph, // functional equivalents. Status FunctionalizeControlFlow(Graph* graph, FunctionLibraryDefinition* library) { - VLOG(2) << "FunctionalizeControlFlow: " + VLOG(2) << "FunctionalizeControlFlow (initial): " << dump_graph::DumpGraphToFile("functionalize_initial", *graph); // Note: BuildControlFlowInfo() requires that the graph's source node is // connected to all source nodes in the graph. Many graphs violate this @@ -1448,7 +1446,11 @@ Status FunctionalizeControlFlow(Graph* graph, // FunctionalizeControlFlow is invoked for every function, so the loops's // bodies and conditionals that were extracted into functions will be handled // in successive invocations. - return FunctionalizeCond::Functionalize(graph, library); + TF_RETURN_IF_ERROR(FunctionalizeCond::Functionalize(graph, library)); + + VLOG(2) << "FunctionalizeControlFlow (final): " + << dump_graph::DumpGraphToFile("functionalize_final", *graph); + return Status::OK(); } } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc index 4acdf1a26d..01d2b28275 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc @@ -110,7 +110,7 @@ TEST(FunctionalizeControlFlow, Conditional) { auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32); auto less = ops::Less(scope.WithOpName("cond/Less"), y, x); auto if_op = ops::XlaIf(scope.WithOpName("cond/Merge_If"), less, - std::initializer_list{x, y, less}, then_fn, + std::initializer_list{less, y, x}, then_fn, else_fn, {DT_INT32}); GraphDef expected; TF_EXPECT_OK(scope.ToGraphDef(&expected)); @@ -120,10 +120,10 @@ TEST(FunctionalizeControlFlow, Conditional) { // then body. { Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2); - auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_2); + auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0); auto cond = ops::Const( scope.WithOpName("cond").WithControlDependencies(identity), 17); auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond); @@ -136,20 +136,20 @@ TEST(FunctionalizeControlFlow, Conditional) { TF_EXPECT_OK(InstantiateFunctionForTest(then_fn.name(), library, &result)); EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types); + EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types); TF_EXPECT_GRAPH_EQ(expected, result.gdef); } // else body. { Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2); - auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_2); + auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0); auto cond_1 = ops::Const( scope.WithOpName("cond_1").WithControlDependencies(identity), 23); - auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_0, cond_1); + auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1); auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); GraphDef expected; @@ -159,7 +159,7 @@ TEST(FunctionalizeControlFlow, Conditional) { TF_EXPECT_OK(InstantiateFunctionForTest(else_fn.name(), library, &result)); EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types); + EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types); TF_EXPECT_GRAPH_EQ(expected, result.gdef); } } -- GitLab From 7fd47e4d2de009eba0698e7b5f65ae899f6f6624 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 16 Oct 2017 14:34:38 -0700 Subject: [PATCH 038/573] Enable C API for gradients_test.py PiperOrigin-RevId: 172379338 --- tensorflow/python/ops/gradients_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py index de3dd03486..f0cffbab30 100644 --- a/tensorflow/python/ops/gradients_test.py +++ b/tensorflow/python/ops/gradients_test.py @@ -78,6 +78,7 @@ def _OpsBetween(graph, to_ops, from_ops): return between_ops +@test_util.with_c_api class GradientsTest(test_util.TensorFlowTestCase): def _OpNames(self, op_list): @@ -264,6 +265,10 @@ class GradientsTest(test_util.TensorFlowTestCase): self.assertEqual(10.0, grads[1].eval()) def testNoGradientForStringOutputs(self): + # This test can't be run twice because the TestStringOutput gradient can + # only be registered once. Just run with the C API enabled. + if not ops._USE_C_API: return + with ops.Graph().as_default(): def _TestOpGrad(_, float_grad, string_grad): @@ -409,6 +414,7 @@ class GradientsTest(test_util.TensorFlowTestCase): np.testing.assert_allclose(a, b) +@test_util.with_c_api class FunctionGradientsTest(test_util.TensorFlowTestCase): @classmethod @@ -498,6 +504,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): f.add_to_graph(ops.Graph()) +@test_util.with_c_api class StopGradientTest(test_util.TensorFlowTestCase): def testStopGradient(self): @@ -508,6 +515,7 @@ class StopGradientTest(test_util.TensorFlowTestCase): assert igrad is None +@test_util.with_c_api class PreventGradientTest(test_util.TensorFlowTestCase): def testPreventGradient(self): @@ -518,6 +526,7 @@ class PreventGradientTest(test_util.TensorFlowTestCase): _ = gradients.gradients(out, inp) +@test_util.with_c_api class HessianVectorProductTest(test_util.TensorFlowTestCase): def testHessianVectorProduct(self): @@ -546,6 +555,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase): self.assertAllClose(hess_v_value, hess_v_actual) +@test_util.with_c_api class HessianTest(test_util.TensorFlowTestCase): def testHessian1D(self): @@ -594,6 +604,7 @@ class HessianTest(test_util.TensorFlowTestCase): gradients.hessians(x, x) +@test_util.with_c_api class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): def testIndexedSlicesToTensor(self): @@ -651,6 +662,9 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): c_sparse = ops.IndexedSlices( array_ops.placeholder(dtypes.float32), array_ops.placeholder(dtypes.int32), constant([100, 100, 100, 100])) + # "always" filter prevents the warning from being suppressed if it was + # already triggered in a different test. + warnings.simplefilter("always") with warnings.catch_warnings(record=True) as w: math_ops.multiply(c_sparse, 1.0) self.assertEqual(1, len(w)) @@ -671,6 +685,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): str(w[0].message)) +@test_util.with_c_api class OnlyRealGradientsTest(test_util.TensorFlowTestCase): def testRealOnly(self): -- GitLab From afaacfdb4bbece8acc03b0456cfe2819db01f5c8 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Mon, 16 Oct 2017 14:43:07 -0700 Subject: [PATCH 039/573] Default to procuring ResourceVariables in variable_scope.variable when use_resource is not set and Eager mode is enabled. PiperOrigin-RevId: 172380659 --- tensorflow/python/ops/variable_scope.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 87805b5171..4614110ba6 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -1927,11 +1927,17 @@ def variable(initial_value=None, caching_device=None, name=None, dtype=None): - if get_variable_scope().use_resource: + use_resource = get_variable_scope().use_resource + if use_resource or (use_resource is None and context.in_eager_mode()): return resource_variable_ops.ResourceVariable( initial_value=initial_value, trainable=trainable, collections=collections, validate_shape=validate_shape, caching_device=caching_device, name=name, dtype=dtype) + elif not use_resource and context.in_eager_mode(): + raise RuntimeError( + "VariableScope should use resource variable in Eager mode, but " + "use_resource is False." + ) else: return variables.Variable( initial_value=initial_value, trainable=trainable, -- GitLab From 4e1d0f4e32b7f7a463930543dc773997bdb9d545 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 16 Oct 2017 14:57:15 -0700 Subject: [PATCH 040/573] Fix broken link in debugger doc (#13757) --- tensorflow/docs_src/programmers_guide/debugger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md index 3ede42e8f7..3f9f155457 100644 --- a/tensorflow/docs_src/programmers_guide/debugger.md +++ b/tensorflow/docs_src/programmers_guide/debugger.md @@ -440,7 +440,7 @@ accuracy_score = classifier.evaluate(x=test_set.data, [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py), -based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to +based on @{$tflearn$tf-learn's iris tutorial}, contains a full example of how to use the tfdbg with `Estimator`s. To run this example, do: ```none -- GitLab From 7b6eec7e1175624458a48945bba3f6400e754d33 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Mon, 16 Oct 2017 15:25:22 -0700 Subject: [PATCH 041/573] Add cc file with definition of tensorflow::gtl::nullopt. If you ODR-use nullopt, you currently get a linker error. Oops. PiperOrigin-RevId: 172387553 --- tensorflow/core/lib/gtl/optional.cc | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tensorflow/core/lib/gtl/optional.cc diff --git a/tensorflow/core/lib/gtl/optional.cc b/tensorflow/core/lib/gtl/optional.cc new file mode 100644 index 0000000000..8dea073788 --- /dev/null +++ b/tensorflow/core/lib/gtl/optional.cc @@ -0,0 +1,25 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/lib/gtl/optional.h" + +namespace tensorflow { +namespace gtl { + +nullopt_t::init_t nullopt_t::init; +extern const nullopt_t nullopt{nullopt_t::init}; + +} // namespace gtl +} // namespace tensorflow -- GitLab From dc442f4ce2d3b11b56721337fe2b9e2282be93be Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 16 Oct 2017 15:29:59 -0700 Subject: [PATCH 042/573] Add return_nodes option to ImportGraphDef The is similar to the return_tensors option. return_tensors cannot be used to fetch nodes with no outputs, so return_nodes is necessary. In addition, this change also refactors the ImportGraphDef signature to return all optional return values in a single struct. This is to keep the ImportGraphDef signature from getting too long, and also makes the call sites simpler. PiperOrigin-RevId: 172388270 --- tensorflow/c/c_api.cc | 18 +- tensorflow/c/while_loop_test.cc | 6 +- tensorflow/core/graph/graph_constructor.cc | 73 +++++-- tensorflow/core/graph/graph_constructor.h | 60 ++++-- .../core/graph/graph_constructor_test.cc | 191 ++++++++++++------ 5 files changed, 240 insertions(+), 108 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 334f867e47..79fbd8c90c 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -1854,18 +1854,18 @@ static void GraphImportGraphDefLocked(TF_Graph* graph, const GraphDef& def, return; } const int last_node_id = graph->graph.num_node_ids(); - std::vector> return_outputs_vec; - status->status = tensorflow::ImportGraphDef( - opts->opts, def, &graph->graph, &graph->refiner, &return_outputs_vec); + tensorflow::ImportGraphDefResults results; + status->status = tensorflow::ImportGraphDef(opts->opts, def, &graph->graph, + &graph->refiner, &results); if (!status->status.ok()) return; for (int i = last_node_id; i < graph->graph.num_node_ids(); ++i) { auto* node = graph->graph.FindNodeId(i); if (node != nullptr) graph->name_map[node->name()] = node; } - DCHECK_EQ(return_outputs_vec.size(), num_return_outputs); + DCHECK_EQ(results.return_tensors.size(), num_return_outputs); for (int i = 0; i < num_return_outputs; ++i) { - return_outputs[i].oper = ToOperation(return_outputs_vec[i].first); - return_outputs[i].index = return_outputs_vec[i].second; + return_outputs[i].oper = ToOperation(results.return_tensors[i].first); + return_outputs[i].index = results.return_tensors[i].second; } } @@ -1945,11 +1945,11 @@ Status CopyGraph(Graph* src_graph, Graph* dst_graph, } // TOOD(skyewm): change to OutputTensor - std::vector> return_tensors; + tensorflow::ImportGraphDefResults results; TF_RETURN_IF_ERROR( - ImportGraphDef(opts, gdef, dst_graph, dst_refiner, &return_tensors)); + ImportGraphDef(opts, gdef, dst_graph, dst_refiner, &results)); - for (const auto& pair : return_tensors) { + for (const auto& pair : results.return_tensors) { return_nodes->emplace_back(pair.first, pair.second); } return Status::OK(); diff --git a/tensorflow/c/while_loop_test.cc b/tensorflow/c/while_loop_test.cc index 2423d83dda..d2d887f32c 100644 --- a/tensorflow/c/while_loop_test.cc +++ b/tensorflow/c/while_loop_test.cc @@ -318,7 +318,7 @@ TEST_F(CApiWhileLoopTest, InvalidCondOutputNode) { // TODO(skyewm): this error message could be more informative. Add explicit // checks for this case in the while loop implementation? ExpectError(TF_INVALID_ARGUMENT, - "Requested return node 'p0' not found in graph def"); + "Requested return tensor 'p0:0' not found in graph def"); } TEST_F(CApiWhileLoopTest, InvalidCondOutputIndex) { @@ -358,7 +358,7 @@ TEST_F(CApiWhileLoopTest, InvalidBodyOutputNode) { // TODO(skyewm): this error message could be more informative. Add explicit // checks for this case in the while loop implementation? ExpectError(TF_INVALID_ARGUMENT, - "Requested return node 'p0' not found in graph def"); + "Requested return tensor 'p0:0' not found in graph def"); } // TODO(skyewm): enable this when it works (currently segfaults!) @@ -389,7 +389,7 @@ TEST_F(CApiWhileLoopTest, WrongGraph) { params_->body_outputs[0] = inputs_[0]; // TODO(skyewm): improve error message ExpectError(TF_INVALID_ARGUMENT, - "Requested return node 'p0' not found in graph def"); + "Requested return tensor 'p0:0' not found in graph def"); } TEST_F(CApiWhileLoopTest, BadTypes) { diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index 15f7b9fe8c..92b4843221 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -79,6 +79,7 @@ class GraphConstructor { skip_mapped_nodes(in.skip_mapped_nodes), control_dependencies(in.control_dependencies), return_tensors(in.return_tensors), + return_nodes(in.return_nodes), importing(true) {} bool allow_internal_ops; @@ -89,6 +90,7 @@ class GraphConstructor { bool skip_mapped_nodes; std::vector control_dependencies; std::vector return_tensors; + std::vector return_nodes; // TODO(ashankar): This bool exists to separate out functionality required // to make ImportGraphDef a close equivalent of Python's import_graph_def @@ -109,6 +111,7 @@ class GraphConstructor { const FunctionDefLibrary* library, Graph* g, ShapeRefiner* refiner, std::vector>* return_tensors, + std::vector* return_nodes, std::vector* unused_input_map_keys) { if (versions) { TF_RETURN_IF_ERROR(CheckVersions(*versions, TF_GRAPH_DEF_VERSION, @@ -116,7 +119,7 @@ class GraphConstructor { "GraphDef", "graph")); } GraphConstructor c(opts, node_defs, versions, library, g, refiner, - return_tensors, unused_input_map_keys); + return_tensors, return_nodes, unused_input_map_keys); const Status s = c.TryImport(); if (!s.ok()) c.Undo(); return s; @@ -128,6 +131,7 @@ class GraphConstructor { const FunctionDefLibrary* library, Graph* g, ShapeRefiner* refiner, std::vector>* return_tensors, + std::vector* return_nodes, std::vector* unused_input_map_keys) : opts_(opts), node_defs_(node_defs), @@ -137,6 +141,7 @@ class GraphConstructor { original_versions_(g->versions()), refiner_(refiner), return_tensors_(return_tensors), + return_nodes_(return_nodes), unused_input_map_keys_(unused_input_map_keys) {} Status TryImport() { @@ -148,6 +153,7 @@ class GraphConstructor { TF_RETURN_IF_ERROR(AddBackEdges()); TF_RETURN_IF_ERROR(UpdateVersionDef()); TF_RETURN_IF_ERROR(PopulateReturnTensors()); + TF_RETURN_IF_ERROR(PopulateReturnNodes()); FixupSourceAndSinkEdges(g_); return Status::OK(); } @@ -160,6 +166,7 @@ class GraphConstructor { Status AddBackEdges(); Status UpdateVersionDef(); Status PopulateReturnTensors(); + Status PopulateReturnNodes(); void Undo(); @@ -196,6 +203,9 @@ class GraphConstructor { // May be null. Not owned. std::vector>* return_tensors_; + // May be null. Not owned. + std::vector* return_nodes_; + // May be null. Not owned. std::vector* unused_input_map_keys_; @@ -913,7 +923,8 @@ Status GraphConstructor::PopulateReturnTensors() { // Locate id in imported nodes auto iter = gdef_nodes_.find(id.first); if (iter == gdef_nodes_.end()) { - return errors::InvalidArgument("Requested return node '", id.first, + return errors::InvalidArgument("Requested return tensor '", + id.ToString(), "' not found in graph def"); } int num_outputs = iter->second.node->num_outputs(); @@ -935,6 +946,19 @@ Status GraphConstructor::PopulateReturnTensors() { return Status::OK(); } +Status GraphConstructor::PopulateReturnNodes() { + if (opts_.return_nodes.empty()) return Status::OK(); + for (StringPiece name : opts_.return_nodes) { + auto iter = gdef_nodes_.find(name); + if (iter == gdef_nodes_.end()) { + return errors::InvalidArgument("Requested return node '", name, + "' not found in graph def"); + } + return_nodes_->push_back(iter->second.node); + } + return Status::OK(); +} + void GraphConstructor::Undo() { for (const auto& iter : gdef_nodes_) { if (iter.second.node != nullptr) { @@ -965,7 +989,8 @@ Status ConvertGraphDefToGraph(const GraphConstructorOptions& opts, ShapeRefiner refiner(gdef.versions().producer(), g->op_registry()); return GraphConstructor::Construct( opts, gdef.node(), &gdef.versions(), &gdef.library(), g, &refiner, - /*return_tensors=*/nullptr, /*unused_input_map_keys=*/nullptr); + /*return_tensors=*/nullptr, /*return_nodes=*/nullptr, + /*unused_input_map_keys=*/nullptr); } Status ConvertNodeDefsToGraph(const GraphConstructorOptions& opts, @@ -978,31 +1003,40 @@ Status ConvertNodeDefsToGraph(const GraphConstructorOptions& opts, } return GraphConstructor::Construct(opts, node_defs, nullptr, nullptr, g, &refiner, /*return_tensors=*/nullptr, + /*return_nodes=*/nullptr, /*unused_input_map_keys=*/nullptr); } Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef, Graph* g, ShapeRefiner* refiner, - std::vector>* return_tensors, - std::vector* unused_input_map_keys) { + ImportGraphDefResults* results) { if (!opts.return_tensors.empty()) { - if (return_tensors == nullptr) { + if (results == nullptr) { return errors::InvalidArgument( - "return_tensors argument to ImportGraphDef() must be non-null if " + "results argument to ImportGraphDef() must be non-null if " "opts.return_tensors is non-empty"); } - if (!return_tensors->empty()) { + } + + if (!opts.return_nodes.empty()) { + if (opts.skip_mapped_nodes) { + return errors::InvalidArgument( + "Requesting return_nodes with skip_mapped_nodes set is not currently " + "supported"); + } + if (results == nullptr) { return errors::InvalidArgument( - "return_tensors argument to ImportGraphDef() should be empty (has " - "size ", - return_tensors->size(), ")"); + "results argument to ImportGraphDef() must be non-null if " + "opts.return_nodes is non-empty"); } } - if (unused_input_map_keys != nullptr && !unused_input_map_keys->empty()) { - return errors::InvalidArgument( - "If non-null, unused_input_map_keys argument to ImportGraphDef() should" - " be empty (has size ", - unused_input_map_keys->size(), ")"); + + if (results != nullptr) { + if (!results->return_tensors.empty() || !results->return_nodes.empty() || + !results->unused_input_map_keys.empty()) { + return errors::InvalidArgument( + "All fields in results argument to ImportGraphDef() must be empty."); + } } ShapeRefiner default_refiner(gdef.versions().producer(), g->op_registry()); @@ -1034,9 +1068,10 @@ Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef, refiner->set_graph_def_version( std::min(refiner->graph_def_version(), gdef.versions().producer())); - return GraphConstructor::Construct(opts, gdef.node(), &gdef.versions(), - &gdef.library(), g, refiner, - return_tensors, unused_input_map_keys); + return GraphConstructor::Construct( + opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner, + &results->return_tensors, &results->return_nodes, + &results->unused_input_map_keys); } void CopyGraph(const Graph& src, Graph* dest) { diff --git a/tensorflow/core/graph/graph_constructor.h b/tensorflow/core/graph/graph_constructor.h index a8f9f2b245..6cd9347d96 100644 --- a/tensorflow/core/graph/graph_constructor.h +++ b/tensorflow/core/graph/graph_constructor.h @@ -72,8 +72,6 @@ struct ImportGraphDefOptions { // used to create the existing nodes referenced in `input_map`. // TODO(skyewm): can we remove this requirement? How do we access the original // shape refiner? - // - // TODO(skyewm): add functionality to retrieve unused `input_map` keys std::map input_map; // If true, nodes that will have all output edges removed because of @@ -88,10 +86,10 @@ struct ImportGraphDefOptions { // other nodes in `gdef`. std::vector control_dependencies; - // Tensors in `gdef` that will be returned via the `return_tensors` output - // parameter of `ImportGraphDef()`. If this list is non-empty, the caller must - // pass an empty vector to `ImportGraphDef()`. The vector will be populated - // with the imported nodes in `g`. + // Tensors in `gdef` that will be returned via the ImportGraphDefResults + // output parameter of `ImportGraphDef()`. If this list is non-empty, the + // caller must pass a results object to `ImportGraphDef()`. The + // `return_tensors` field will be populated with the imported nodes in `g`. // // Entries should not include `prefix`, i.e., each TensorId's name should be // the name as it originally appears in `gdef`. @@ -100,12 +98,43 @@ struct ImportGraphDefOptions { // corresponding existing tensor in `g` will be returned. std::vector return_tensors; + // The names of nodes in `gdef` that will be returned via the + // ImportGraphDefResults output parameter of `ImportGraphDef()`. If this list + // is non-empty, the caller must pass a results object to + // `ImportGraphDef()`. The `return_nodes` field will be populated with the + // imported nodes in `g`. + // + // Entries should not include `prefix`, i.e., each node's name should be the + // name as it originally appears in `gdef`. + // + // Unlike `return_tensors`, `input_map` has no effect on the nodes + // returned. `return_nodes` must be empty if `skip_mapped_nodes` is true. + // TODO(skyewm): make this work with `skip_mapped_nodes` if there's a need. + std::vector return_nodes; + // TODO(ashankar): Enable handling of GraphDefs produced by newer binaries // with ops that are not defined in the binary calling ImportGraphDef. // Similar to the producer_op_list argument to import_graph_def in the // python API. }; +// Optional results that may be returned by ImportGraphDef. +struct ImportGraphDefResults { + // The requested tensors associated with + // ImportGraphDefOptions::return_tensors. Note that the index may be different + // than the requested index if the returned tensor has been remapped according + // to `input_map`. + typedef int Index; + std::vector> return_tensors; + + // The requested nodes associated with ImportGraphDefOptions::return_nodes. + std::vector return_nodes; + + // Keys in ImportGraphDefOptions::input_map that weren't used as an input to + // any node in`gdef`. + std::vector unused_input_map_keys; +}; + // Adds the graph in GraphDef `gdef` into an existing Graph `*g`. // // On error, returns non-OK and leaves `*g` unmodified. @@ -115,21 +144,16 @@ struct ImportGraphDefOptions { // allows the caller to validate shapes of those nodes (since // ShapeRefiner::AddNode must be called in topological order). // -// Each `return_tensors` entry is the requested node and output index. The index -// is included in case the returned tensor has been remapped according to -// `input_map`. -// -// If `unused_input_map_keys` is non-null, it should be empty and will be -// populated with any keys in `opts.input_map` that aren't used as an input to -// any node in `gdef`. +// `results` must be non-null if `opts.return_tensors` or `opts.result_nodes` is +// non-empty. It can also be set to fetch the unused input map keys. If it's +// non-null, all the vector fields must be empty. // // TODO(ashankar): Push this mechanism and get rid of Session::Extend() // as a means of enhancing an existing Graph. -extern Status ImportGraphDef( - const ImportGraphDefOptions& opts, const GraphDef& gdef, Graph* g, - ShapeRefiner* refiner, - std::vector>* return_tensors = nullptr, - std::vector* unused_input_map_keys = nullptr); +extern Status ImportGraphDef(const ImportGraphDefOptions& opts, + const GraphDef& gdef, Graph* g, + ShapeRefiner* refiner, + ImportGraphDefResults* results = nullptr); // Make a copy of "src" into "*dest". // diff --git a/tensorflow/core/graph/graph_constructor_test.cc b/tensorflow/core/graph/graph_constructor_test.cc index f88d707ec5..5242c56ce6 100644 --- a/tensorflow/core/graph/graph_constructor_test.cc +++ b/tensorflow/core/graph/graph_constructor_test.cc @@ -71,14 +71,12 @@ class GraphConstructorTest : public ::testing::Test { void ExpectError(const string& gdef_ascii, const ImportGraphDefOptions& opts, const std::vector& expected_error_strs, ShapeRefiner* refiner = nullptr, - std::vector>* return_tensors = nullptr, - std::vector* unused_input_map_keys = nullptr) { + ImportGraphDefResults* results = nullptr) { // Used to verify that errors don't change graph const string original_graph_description = GraphDebugString(); Convert(gdef_ascii); - Status status = ImportGraphDef(opts, gdef_, &graph_, refiner, - return_tensors, unused_input_map_keys); + Status status = ImportGraphDef(opts, gdef_, &graph_, refiner, results); EXPECT_FALSE(status.ok()); for (const string& error : expected_error_strs) { @@ -97,11 +95,9 @@ class GraphConstructorTest : public ::testing::Test { void ExpectOK(const string& gdef_ascii, const ImportGraphDefOptions& opts, ShapeRefiner* refiner = nullptr, - std::vector>* return_tensors = nullptr, - std::vector* unused_input_map_keys = nullptr) { + ImportGraphDefResults* results = nullptr) { Convert(gdef_ascii); - Status s = ImportGraphDef(opts, gdef_, &graph_, refiner, return_tensors, - unused_input_map_keys); + Status s = ImportGraphDef(opts, gdef_, &graph_, refiner, results); EXPECT_EQ(Status::OK(), s) << s; } @@ -1440,26 +1436,25 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapDuplicateNodeNames) { TEST_F(GraphConstructorTest, ImportGraphDef_InputMapUnusedKeys) { ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); - std::vector unused_input_map_keys; - // No input map ImportGraphDefOptions opts; + ImportGraphDefResults results; ExpectOK( "node { name: 'W1' op: 'TestParams' }" "node { name: 'input' op: 'TestInput' }", - opts, &refiner, nullptr, &unused_input_map_keys); - EXPECT_TRUE(unused_input_map_keys.empty()); + opts, &refiner, &results); + EXPECT_TRUE(results.unused_input_map_keys.empty()); // Non-empty unused_input_map_keys - unused_input_map_keys.push_back(TensorId()); - ExpectError("node { name: 'W2' op: 'TestParams' }", opts, - {"If non-null, unused_input_map_keys argument to ImportGraphDef()" - " should be empty (has size 1)"}, - &refiner, nullptr, &unused_input_map_keys); + results.unused_input_map_keys.push_back(TensorId()); + ExpectError( + "node { name: 'W2' op: 'TestParams' }", opts, + {"All fields in results argument to ImportGraphDef() must be empty."}, + &refiner, &results); // Input map with some used, some unused keys const int kControlSlot = Graph::kControlSlot; - unused_input_map_keys.clear(); + results.unused_input_map_keys.clear(); opts.input_map[TensorId("W2", kControlSlot)] = TensorId("W1", kControlSlot); opts.input_map[TensorId("new_input", 0)] = TensorId("input", 0); opts.input_map[TensorId("new_input", 1)] = TensorId("input", 0); @@ -1473,11 +1468,11 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapUnusedKeys) { node { name: 't1' op: 'TestMul' input: [ 'new_input:0', 'new_input:1' ] } node { name: 't2' op: 'TestMul' input: [ 't1:0', 't1:0' ] } )EOF", - opts, &refiner, nullptr, &unused_input_map_keys); + opts, &refiner, &results); std::vector expected_unused_keys = { TensorId("new_input", kControlSlot), TensorId("t1", 1)}; - EXPECT_EQ(unused_input_map_keys, expected_unused_keys); + EXPECT_EQ(results.unused_input_map_keys, expected_unused_keys); } TEST_F(GraphConstructorTest, ImportGraphDef_SkipMappedNodes_FullyMapped) { @@ -1567,11 +1562,11 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensors) { opts.return_tensors.push_back({"input", 1}); opts.return_tensors.push_back({"t1", 0}); opts.return_tensors.push_back({"input", 0}); - std::vector> return_tensors; + ImportGraphDefResults results; ExpectOK( "node { name: 'input' op: 'TestInput' }" "node { name: 't1' op: 'TestMul' input: ['input:0', 'input:1'] }", - opts, &refiner, &return_tensors); + opts, &refiner, &results); // Sanity checks EXPECT_TRUE(HasNode("input")); @@ -1580,74 +1575,70 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensors) { EXPECT_TRUE(HasEdge("input", 1, "t1", 1)); // Check return tensors - ASSERT_EQ(return_tensors.size(), 3); - EXPECT_EQ(return_tensors[0].first->name(), "input"); - EXPECT_EQ(return_tensors[0].second, 1); - EXPECT_EQ(return_tensors[1].first->name(), "t1"); - EXPECT_EQ(return_tensors[1].second, 0); - EXPECT_EQ(return_tensors[2].first->name(), "input"); - EXPECT_EQ(return_tensors[2].second, 0); + ASSERT_EQ(results.return_tensors.size(), 3); + EXPECT_EQ(results.return_tensors[0].first->name(), "input"); + EXPECT_EQ(results.return_tensors[0].second, 1); + EXPECT_EQ(results.return_tensors[1].first->name(), "t1"); + EXPECT_EQ(results.return_tensors[1].second, 0); + EXPECT_EQ(results.return_tensors[2].first->name(), "input"); + EXPECT_EQ(results.return_tensors[2].second, 0); // Test using prefix and returning element from input_map opts.return_tensors.clear(); - return_tensors.clear(); + results = ImportGraphDefResults(); opts.prefix = "import"; opts.input_map[{"new_input", 1}] = {"input", 0}; opts.return_tensors.push_back({"new_input", 0}); opts.return_tensors.push_back({"new_input", 1}); ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner, - &return_tensors); + &results); EXPECT_TRUE(HasNode("import/new_input")); - ASSERT_EQ(return_tensors.size(), 2); - EXPECT_EQ(return_tensors[0].first->name(), "import/new_input"); - EXPECT_EQ(return_tensors[0].second, 0); - EXPECT_EQ(return_tensors[1].first->name(), "input"); - EXPECT_EQ(return_tensors[1].second, 0); + ASSERT_EQ(results.return_tensors.size(), 2); + EXPECT_EQ(results.return_tensors[0].first->name(), "import/new_input"); + EXPECT_EQ(results.return_tensors[0].second, 0); + EXPECT_EQ(results.return_tensors[1].first->name(), "input"); + EXPECT_EQ(results.return_tensors[1].second, 0); // Test returning node remapped to source node opts.prefix.clear(); opts.input_map.clear(); opts.return_tensors.clear(); - return_tensors.clear(); + results = ImportGraphDefResults(); opts.input_map[{"new_input", 0}] = {"_SOURCE", 0}; opts.return_tensors.push_back({"new_input", 0}); ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner, - &return_tensors); + &results); EXPECT_TRUE(HasNode("new_input")); - ASSERT_EQ(return_tensors.size(), 1); - EXPECT_EQ(return_tensors[0].first->name(), "_SOURCE"); - EXPECT_EQ(return_tensors[0].second, 0); + ASSERT_EQ(results.return_tensors.size(), 1); + EXPECT_EQ(results.return_tensors[0].first->name(), "_SOURCE"); + EXPECT_EQ(results.return_tensors[0].second, 0); } TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensorsErrors) { - // Passing in return_tensors with empty opts.return_tensors is OK + // Null results with non-empty opts.return_tensors ImportGraphDefOptions opts; - std::vector> return_tensors; - ExpectOK("node { name: 'input' op: 'TestInput' }", opts, nullptr, - &return_tensors); - - // Null return_tensors with non-empty opts.return_tensors opts.return_tensors.push_back({"new_input", 0}); ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, - {"return_tensors argument to ImportGraphDef() must be non-null " - "if opts.return_tensors is non-empty"}); + {"results argument to ImportGraphDef() must be non-null if " + "opts.return_tensors is non-empty"}); - // Non-empty return_tensors - return_tensors.push_back({nullptr, 0}); - ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, - {"return_tensors argument to ImportGraphDef() should be empty " - "(has size 1)"}, - nullptr, &return_tensors); + // Non-empty results.return_tensors + ImportGraphDefResults results; + results.return_tensors.push_back({nullptr, 0}); + ExpectError( + "node { name: 'new_input' op: 'TestInput' }", opts, + {"All fields in results argument to ImportGraphDef() must be empty."}, + nullptr, &results); // Requesting tensor that isn't in graph def - return_tensors.clear(); + results.return_tensors.clear(); ExpectError("node { name: 'W1' op: 'TestParams' }", opts, - {"Requested return node 'new_input' not found in graph def"}, - nullptr, &return_tensors); + {"Requested return tensor 'new_input:0' not found in graph def"}, + nullptr, &results); // Requesting invalid node index opts.return_tensors.clear(); @@ -1655,7 +1646,89 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensorsErrors) { ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, {"Invalid return output 2 of node 'new_input', which has 2 " "output(s)"}, - nullptr, &return_tensors); + nullptr, &results); +} + +TEST_F(GraphConstructorTest, ImportGraphDef_ReturnNodes) { + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); + + ImportGraphDefOptions opts; + opts.return_nodes.push_back("input"); + opts.return_nodes.push_back("t1"); + ImportGraphDefResults results; + ExpectOK( + "node { name: 'input' op: 'TestInput' }" + "node { name: 'input2' op: 'TestInput' }" + "node { name: 't1' op: 'TestMul' input: ['input:0', 'input2:1'] }", + opts, &refiner, &results); + + // Sanity checks + EXPECT_TRUE(HasNode("input")); + EXPECT_TRUE(HasNode("input2")); + EXPECT_TRUE(HasNode("t1")); + EXPECT_TRUE(HasEdge("input", 0, "t1", 0)); + EXPECT_TRUE(HasEdge("input2", 1, "t1", 1)); + + // Check return tensors + ASSERT_EQ(results.return_nodes.size(), 2); + EXPECT_EQ(results.return_tensors.size(), 0); + EXPECT_EQ(results.unused_input_map_keys.size(), 0); + EXPECT_EQ(results.return_nodes[0]->name(), "input"); + EXPECT_EQ(results.return_nodes[1]->name(), "t1"); + + // Test using prefix + opts = ImportGraphDefOptions(); + results = ImportGraphDefResults(); + opts.prefix = "import"; + opts.return_nodes.push_back("input"); + ExpectOK("node { name: 'input' op: 'TestInput' }", opts, &refiner, &results); + + EXPECT_TRUE(HasNode("import/input")); + + ASSERT_EQ(results.return_nodes.size(), 1); + EXPECT_EQ(results.return_nodes[0]->name(), "import/input"); + + // Test that input_map has no effect + opts = ImportGraphDefOptions(); + results = ImportGraphDefResults(); + opts.input_map[{"new_input", 0}] = {"input", 0}; + opts.return_nodes.push_back("new_input"); + ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner, + &results); + + EXPECT_TRUE(HasNode("new_input")); + + ASSERT_EQ(results.return_nodes.size(), 1); + EXPECT_EQ(results.return_nodes[0]->name(), "new_input"); +} + +TEST_F(GraphConstructorTest, ImportGraphDef_ReturnNodesErrors) { + // Null results with non-empty opts.return_nodes + ImportGraphDefOptions opts; + opts.return_nodes.push_back("new_input"); + ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, + {"results argument to ImportGraphDef() must be non-null if " + "opts.return_nodes is non-empty"}); + + // Non-empty results.return_nodes + ImportGraphDefResults results; + results.return_nodes.push_back(nullptr); + ExpectError( + "node { name: 'new_input' op: 'TestInput' }", opts, + {"All fields in results argument to ImportGraphDef() must be empty."}, + nullptr, &results); + + // Requesting node that isn't in graph def + results.return_nodes.clear(); + ExpectError("node { name: 'W1' op: 'TestParams' }", opts, + {"Requested return node 'new_input' not found in graph def"}, + nullptr, &results); + + // Requesting return_nodes with skip_mapped_nodes not yet implemented + opts.skip_mapped_nodes = true; + ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, + {"Requesting return_nodes with skip_mapped_nodes set is not " + "currently supported"}); } TEST_F(GraphConstructorTest, ImportGraphDef_WithCycle) { -- GitLab From 01c76110eb3cb1c378c9d7a14ca9f838bad6c7d1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 15:37:37 -0700 Subject: [PATCH 043/573] Uses head.name in name_scope. This improves the graph naming for MultiHead. PiperOrigin-RevId: 172389494 --- .../contrib/estimator/python/estimator/head.py | 10 +++++----- tensorflow/python/estimator/canned/head.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index e7fe454fbf..f8648fe5bf 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -59,7 +59,7 @@ def multi_class_head(n_classes, `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for multi class classification. @@ -98,7 +98,7 @@ def binary_classification_head( `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for binary classification. @@ -129,7 +129,7 @@ def regression_head(weight_column=None, of the last dimension of the labels `Tensor` (typically, this has shape `[batch_size, label_dimension]`). name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for linear regression. @@ -172,7 +172,7 @@ def multi_label_head(n_classes, string type and have any value in `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for multi-label classification. @@ -272,7 +272,7 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py index b796a3f954..beafe0d5c4 100644 --- a/tensorflow/python/estimator/canned/head.py +++ b/tensorflow/python/estimator/canned/head.py @@ -361,7 +361,7 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes, `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for multi class classification. @@ -453,7 +453,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): logits = _check_logits(logits, self.logits_dimension) # Predict. @@ -562,7 +562,7 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss( `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `Head` for binary classification. @@ -702,7 +702,7 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): with ops.name_scope(None, 'predictions', (logits,)): pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) @@ -802,7 +802,7 @@ def _regression_head_with_mean_squared_error_loss(weight_column=None, of the last dimension of the labels `Tensor` (typically, this has shape `[batch_size, label_dimension]`). name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for linear regression. @@ -846,7 +846,7 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head): self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: -- GitLab From 51e5b692a5f8f6942cb43291ba9faab39e4b6104 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Mon, 16 Oct 2017 15:46:36 -0700 Subject: [PATCH 044/573] Fix ambiguous type comparison in s3_crypto.cc (#13758) tensorflow/contrib/s3/s3_crypto.cc(74): error C2666: 'std::fpos<_Mbstatet>::operator ==': 3 overloads have similar conversions could be 'bool std::fpos<_Mbstatet>::operator ==(std::streamoff) const' or 'bool std::fpos<_Mbstatet>::operator ==(const std::fpos<_Mbstatet> &) We were seeing this compilation error on Windows builds. --- tensorflow/contrib/s3/s3_crypto.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/s3/s3_crypto.cc b/tensorflow/contrib/s3/s3_crypto.cc index 1450384dc0..bbd66371e4 100644 --- a/tensorflow/contrib/s3/s3_crypto.cc +++ b/tensorflow/contrib/s3/s3_crypto.cc @@ -71,7 +71,7 @@ class S3Sha256OpenSSLImpl : public Aws::Utils::Crypto::Hash { SHA256_Init(&sha256); auto currentPos = stream.tellg(); - if (currentPos == -1) { + if (currentPos == std::streampos(std::streamoff(-1))) { currentPos = 0; stream.clear(); } -- GitLab From a072aa0d2c8412160748995bdea0cc15f121fd95 Mon Sep 17 00:00:00 2001 From: Vijay Vasudevan Date: Mon, 16 Oct 2017 15:50:36 -0700 Subject: [PATCH 045/573] Revert "Fix broken link in debugger doc" (#13760) * Revert "Fix broken link in debugger doc (#13757)" This reverts commit 4e1d0f4e32b7f7a463930543dc773997bdb9d545. --- tensorflow/docs_src/programmers_guide/debugger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md index 3f9f155457..3ede42e8f7 100644 --- a/tensorflow/docs_src/programmers_guide/debugger.md +++ b/tensorflow/docs_src/programmers_guide/debugger.md @@ -440,7 +440,7 @@ accuracy_score = classifier.evaluate(x=test_set.data, [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py), -based on @{$tflearn$tf-learn's iris tutorial}, contains a full example of how to +based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to use the tfdbg with `Estimator`s. To run this example, do: ```none -- GitLab From e3b8d3cc2a0099fecdc103f8422b34eda1eaee1f Mon Sep 17 00:00:00 2001 From: michelleirvine Date: Mon, 16 Oct 2017 16:00:55 -0700 Subject: [PATCH 046/573] Update README.md (#13688) * Update README.md Update information about local builds and TensorFlow's CI system. * Update README.md * Update README.md * Update README.md --- tensorflow/tools/ci_build/README.md | 143 ++++++++++------------------ 1 file changed, 52 insertions(+), 91 deletions(-) diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md index ad83669950..acef833909 100644 --- a/tensorflow/tools/ci_build/README.md +++ b/tensorflow/tools/ci_build/README.md @@ -1,115 +1,76 @@ # TensorFlow Builds -This directory contains all the files and setup instructions to run all -the important builds and tests. **You can trivially run it yourself!** It also -run continuous integration [ci.tensorflow.org](https://ci.tensorflow.org). - - +This directory contains all the files and setup instructions to run all the +important builds and tests. You can run it yourself! ## Run It Yourself -1. Install [Docker](http://www.docker.com/). Follow instructions - [on the Docker site](https://docs.docker.com/installation/). - - You can run all the jobs **without docker** if you are on mac or on linux - and you just don't want docker. Just install all the dependencies from - [Installing TensorFlow](https://www.tensorflow.org/install/). - Then run any of the one liners below without the - `tensorflow/tools/ci_build/ci_build.sh` in them. - -2. Clone tensorflow repository. - - ```bash - git clone https://github.com/tensorflow/tensorflow.git - ``` - -3. Go to tensorflow directory - - ```bash - cd tensorflow - ``` - -4. Build what you want, for example - - ```bash - tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... - ``` - If you are using the Docker image on Windows or OS X, the Docker VM's default - memory limit may be too low to build TensorFlow. This can result in - strange-looking errors, e.g. the compilation may fail with `gcc: internal - compiler error: Killed (program cc1plus)`. Try increasing the memory limit in - the Docker preferences. - - -## Jobs - -The jobs run by [ci.tensorflow.org](https://ci.tensorflow.org) include following: - -```bash -# Note: You can run the following one-liners yourself if you have Docker. Run -# without `tensorflow/tools/ci_build/ci_build.sh` on mac or linux without Docker. - -# build and run cpu tests -tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... +You have two options when running TensorFlow tests locally on your +machine. First, using docker, you can run our Continuous Integration +(CI) scripts on tensorflow devel images. The other option is to install +all TensorFlow dependencies on your machine and run the scripts +natively on your system. -# build and run gpu tests (note if you get unstable results you may be running -# out of gpu memory - if so add "--jobs=1" argument) -tensorflow/tools/ci_build/ci_build.sh GPU bazel test -c opt --config=cuda //tensorflow/... +### Run TensorFlow CI Scripts using Docker -# build pip with gpu support -tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda +1. Install Docker following the [instructions on the docker website](https://docs.docker.com/engine/installation/). -# build and run gpu tests using python 3 -CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda +2. Start a container with one of the devel images here: + https://hub.docker.com/r/tensorflow/tensorflow/tags/. -# build android example app -tensorflow/tools/ci_build/ci_build.sh ANDROID tensorflow/tools/ci_build/builds/android.sh +3. Based on your choice of the image, pick one of the scripts under + https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/ci_build/linux + and run them from the TensorFlow repository root. -# cmake cpu build and test -tensorflow/tools/ci_build/ci_build.sh CPU tensorflow/tools/ci_build/builds/cmake.sh +### Run TensorFlow CI Scripts Natively on your Machine -# run bash inside the container -CI_DOCKER_EXTRA_PARAMS='-it --rm' tensorflow/tools/ci_build/ci_build.sh CPU /bin/bash -``` +1. Follow the instructions at https://www.tensorflow.org/install/install_sources, + but stop when you get to the section "Configure the installation". You do not + need to configure the installation to run the CI scripts. -**Note**: The set of jobs and how they are triggered is still evolving. -There are builds for master branch on cpu, gpu and android. There is a build -for incoming gerrit changes. Gpu tests and benchmark are coming soon. Check -[ci.tensorflow.org](https://ci.tensorflow.org) for current jobs. +2. Pick the appropriate OS and python version you have installed, + and run the script under tensorflow/tools/ci_build/. +## TensorFlow Continuous Integration +To verify that new changes don’t break TensorFlow, we run builds and +tests on either [Jenkins](https://jenkins-ci.org/) or a CI system +internal to Google. -## How Does TensorFlow Continuous Integration Work +We can trigger builds and tests on updates to master or on each pull +request. Contact one of the repository maintainers to trigger builds +on your pull request. -We use [jenkins](https://jenkins-ci.org/) as our continuous integration. -It is running at [ci.tensorflow.org](https://ci.tensorflow.org). -All the jobs are run within [docker](http://www.docker.com/) containers. +### View CI Results -Builds can be triggered by push to master, push a change set or manually. -The build started in jenkins will first pull the git tree. Then jenkins builds -a docker container (using one of those Dockerfile.* files in this directory). -The build itself is run within the container itself. +The Pull Request will show if the change passed or failed the checks. -Source tree lives in jenkins job workspace. Docker container for jenkins -are transient - deleted after the build. Containers build very fast thanks -to docker caching. Individual builds are fast thanks to bazel caching. +From the pull request, click **Show all checks** to see the list of builds +and tests. Click on **Details** to see the results from Jenkins or the internal +CI system. +Results from Jenkins are displayed in the Jenkins UI. For more information, +see the [Jenkns documentation](https://jenkins.io/doc/). +Results from the internal CI system are displayed in the Build Status UI. In +this UI, to see the logs for a failed build: -## Implementation Details +* Click on the **INVOCATION LOG** tab to see the invocation log. -* The ci_build.sh script create and run docker container with all dependencies. - The builds/with_the_same_user together with ci_build.sh creates an environment - which is the same inside the container as it is outside. The same user, group, - path, so that docker symlinks work inside and outside the container. You can - use it for your development. Edit files in your git clone directory. If you - run the ci_build.sh it gets this directory mapped inside the container and - build your tree. +* Click on the **ARTIFACTS** tab to see a list of all artifacts, including logs. -* The unusual `bazel-ci_build-cache` directory is mapped to docker container - performing the build using docker's --volume parameter. This way we cache - bazel output between builds. +* Individual test logs may be available. To see these logs, from the **TARGETS** + tab, click on the failed target. Then, click on the **TARGET LOG** tab to see + its test log. -* The `builds` directory within this folder contains shell scripts to run within - the container. They essentially contains workarounds for current limitations - of bazel. + If you’re looking at target that is sharded or a test that is flaky, then + the build tool divided the target into multiple shards or ran the test + multiple times. Each test log is specific to the shard, run, and attempt. + To see a specific log: + + 1. Click on the log icon that is on the right next to the shard, run, + and attempt number. + + 2. In the grid that appears on the right, click on the specific shard, + run, and attempt to view its log. You can also type the desired shard, + run, or attempt number in the field above its grid. -- GitLab From 99dffc958a1cfa4e5a2f81e8f4085277a0c34bd9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 16:34:35 -0700 Subject: [PATCH 047/573] Better error message for eager-specific APIs PiperOrigin-RevId: 172397124 --- tensorflow/contrib/eager/python/saver.py | 10 ++++++++++ tensorflow/python/eager/backprop.py | 4 ++++ tensorflow/python/eager/function_test.py | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py index d289b83f53..2bf11d3f20 100644 --- a/tensorflow/contrib/eager/python/saver.py +++ b/tensorflow/contrib/eager/python/saver.py @@ -19,6 +19,7 @@ from __future__ import print_function import contextlib +from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import resource_variable_ops @@ -64,7 +65,12 @@ def restore_variables_on_create(save_path): Raises: NotFoundError: If the variable is not found in checkpoint. + ValueError: If not used in eager mode. """ + if context.in_graph_mode(): + raise ValueError( + "Currently, restore_variables_on_create can only be used with " + "eager execution enabled.") if save_path: ckpt_var_cache = dict() reader = checkpoint_utils.load_checkpoint(save_path) @@ -102,6 +108,10 @@ class Saver(object): """ def __init__(self, var_list): + if context.in_graph_mode(): + raise ValueError("Currently, tfe.Saver can only be used when eager " + "execution is enabled. Use tf.train.Saver when " + "building graphs.") self._saver = _saver.Saver(var_list=var_list) def save(self, save_path, global_step=None): diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 7f1a770513..1819fba4cb 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -337,6 +337,10 @@ def implicit_val_and_grad(f): end_node = f(*args) variables = tape.top_tape_watched_variables() sources = [x.handle for x in variables] + + if not sources: + raise ValueError("no trainable variables were accessed while the " + "function was being computed.") grad = imperative_grad.imperative_grad(_default_vspace, tape.pop_tape(), nest.flatten(end_node), diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 899b6d59b7..e27f9ebc27 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -31,6 +31,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope class FunctionTest(test.TestCase): @@ -99,6 +100,16 @@ class FunctionTest(test.TestCase): self.assertAllEqual(2, g(constant_op.constant(2)).numpy()) + def testGraphModeEagerGradError(self): + with context.graph_mode(): + def f(): + x = variable_scope.get_variable( + 'v', initializer=constant_op.constant(1.0)) + return x * constant_op.constant(2.0) + with self.assertRaisesRegexp(ValueError, + 'no trainable variables were accessed'): + backprop.implicit_val_and_grad(f)() + def testDefunCallBackpropUsingSameObjectForMultipleArguments(self): @function.defun -- GitLab From 528457ea3cbe4edfbd3eb90c303b2a1408fe8d65 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Oct 2017 17:19:26 -0700 Subject: [PATCH 048/573] Add GPU and CPU implementation of `tf.histogram_fixed_width`. (#13731) * Add GPU and CPU implementation of `tf.histogram_fixed_width`. This fix adds the GPU and CPU implementation of `tf.histogram_fixed_width`. The previous implementation was done in python. This fix adds C++ kernel for GPU and CPU> Signed-off-by: Yong Tang * Update python ops for `tf.histogram_fixed_width` Signed-off-by: Yong Tang * Update test cases to invoke GPU test for `tf.histogram_fixed_width` Signed-off-by: Yong Tang * Disable int64 output on GPU for now as atomicAdd is not supported yet. Signed-off-by: Yong Tang * Address review feedback and use a stable version of summation. Signed-off-by: Yong Tang * Maintain backward compatibility of the API Keep `dtype` and make sure `nbins = 100` is in attr Signed-off-by: Yong Tang --- tensorflow/core/BUILD | 1 + tensorflow/core/kernels/BUILD | 12 ++ tensorflow/core/kernels/histogram_op.cc | 147 ++++++++++++++++++ tensorflow/core/kernels/histogram_op.h | 38 +++++ .../core/kernels/histogram_op_gpu.cu.cc | 125 +++++++++++++++ tensorflow/core/ops/math_ops.cc | 38 +++++ tensorflow/python/ops/histogram_ops.py | 31 +--- tensorflow/python/ops/histogram_ops_test.py | 8 +- 8 files changed, 369 insertions(+), 31 deletions(-) create mode 100644 tensorflow/core/kernels/histogram_op.cc create mode 100644 tensorflow/core/kernels/histogram_op.h create mode 100644 tensorflow/core/kernels/histogram_op_gpu.cu.cc diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index f60c0d76cf..a0c8fae69a 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -781,6 +781,7 @@ cc_library( "//tensorflow/core/kernels:dataset_ops", "//tensorflow/core/kernels:fake_quant_ops", "//tensorflow/core/kernels:function_ops", + "//tensorflow/core/kernels:histogram_op", "//tensorflow/core/kernels:image", "//tensorflow/core/kernels:io", "//tensorflow/core/kernels:linalg", diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 2c02571346..ca5356b6e7 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2499,6 +2499,7 @@ cc_library( ":cross_op", ":cwise_op", ":fft_ops", + ":histogram_op", ":matmul_op", ":population_count_op", ":reduction_ops", @@ -3096,6 +3097,17 @@ tf_kernel_library( ], ) +tf_kernel_library( + name = "histogram_op", + prefix = "histogram_op", + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//third_party/eigen3", + ] + if_cuda(["@cub_archive//:cub"]), +) + tf_kernel_library( name = "l2loss_op", prefix = "l2loss_op", diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc new file mode 100644 index 0000000000..c170f172e4 --- /dev/null +++ b/tensorflow/core/kernels/histogram_op.cc @@ -0,0 +1,147 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/math_ops.cc. + +#define EIGEN_USE_THREADS + +#include "tensorflow/core/kernels/histogram_op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef Eigen::GpuDevice GPUDevice; + +namespace functor { + +template +struct HistogramFixedWidthFunctor { + static Status Compute(OpKernelContext* context, + const typename TTypes::ConstTensor& values, + const typename TTypes::ConstTensor& value_range, + int32 nbins, typename TTypes::Tensor& out) { + const CPUDevice& d = context->eigen_device(); + + Tensor index_to_bin_tensor; + + TF_RETURN_IF_ERROR(context->forward_input_or_allocate_temp( + {0}, DataTypeToEnum::value, TensorShape({values.size()}), + &index_to_bin_tensor)); + auto index_to_bin = index_to_bin_tensor.flat(); + + const double step = static_cast(value_range(1) - value_range(0)) / + static_cast(nbins); + + // The calculation is done by finding the slot of each value in `values`. + // With [a, b]: + // step = (b - a) / nbins + // (x - a) / step + // , then the entries are mapped to output. + index_to_bin.device(d) = + ((values.cwiseMax(value_range(0)) - values.constant(value_range(0))) + .template cast() / + step) + .template cast() + .cwiseMin(nbins - 1); + + out.setZero(); + for (int32 i = 0; i < index_to_bin.size(); i++) { + out(index_to_bin(i)) += Tout(1); + } + return Status::OK(); + } +}; + +} // namespace functor + +template +class HistogramFixedWidthOp : public OpKernel { + public: + explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("nbins", &nbins_)); + OP_REQUIRES( + ctx, (nbins_ > 0), + errors::InvalidArgument("nbins should be a positive number, but got '", + nbins_, "'")); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor& values_tensor = ctx->input(0); + const Tensor& value_range_tensor = ctx->input(1); + + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()), + errors::InvalidArgument("value_range should be a vector.")); + OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2), + errors::InvalidArgument( + "value_range should be a vector of 2 elements.")); + + const auto values = values_tensor.flat(); + const auto value_range = value_range_tensor.flat(); + + OP_REQUIRES( + ctx, (value_range(0) < value_range(1)), + errors::InvalidArgument("value_range should satisfy value_range[0] < " + "value_range[1], but got '[", + value_range(0), ", ", value_range(1), "]'")); + + Tensor* out_tensor; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, TensorShape({nbins_}), &out_tensor)); + auto out = out_tensor->flat(); + + OP_REQUIRES_OK( + ctx, functor::HistogramFixedWidthFunctor::Compute( + ctx, values, value_range, nbins_, out)); + } + + private: + int nbins_; +}; + +#define REGISTER_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("dtype"), \ + HistogramFixedWidthOp) \ + REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("dtype"), \ + HistogramFixedWidthOp) + +TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS); +#undef REGISTER_KERNELS + +#if GOOGLE_CUDA +#define REGISTER_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ + .Device(DEVICE_GPU) \ + .HostMemory("value_range") \ + .TypeConstraint("T") \ + .TypeConstraint("dtype"), \ + HistogramFixedWidthOp) + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_KERNELS); +#undef REGISTER_KERNELS + +#endif // GOOGLE_CUDA + +} // end namespace tensorflow diff --git a/tensorflow/core/kernels/histogram_op.h b/tensorflow/core/kernels/histogram_op.h new file mode 100644 index 0000000000..1b253f7fed --- /dev/null +++ b/tensorflow/core/kernels/histogram_op.h @@ -0,0 +1,38 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_HISTOGRAM_OP_H_ +#define TENSORFLOW_HISTOGRAM_OP_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" + +namespace tensorflow { +namespace functor { + +template +struct HistogramFixedWidthFunctor { + static Status Compute(OpKernelContext* context, + const typename TTypes::ConstTensor& values, + const typename TTypes::ConstTensor& value_range, + int32 nbins, typename TTypes::Tensor& out); +}; + +} // end namespace functor +} // end namespace tensorflow + +#endif // TENSORFLOW_HISTOGRAM_OP_H_ diff --git a/tensorflow/core/kernels/histogram_op_gpu.cu.cc b/tensorflow/core/kernels/histogram_op_gpu.cu.cc new file mode 100644 index 0000000000..c2bb958be8 --- /dev/null +++ b/tensorflow/core/kernels/histogram_op_gpu.cu.cc @@ -0,0 +1,125 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include "tensorflow/core/kernels/histogram_op.h" +#include "external/cub_archive/cub/device/device_histogram.cuh" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + +namespace tensorflow { + +typedef Eigen::GpuDevice GPUDevice; + +namespace functor { + +// TODO(yongtang) int64 of atomicAdd is not supported yet. +template +struct HistogramFixedWidthFunctor { + static Status Compute(OpKernelContext* context, + const typename TTypes::ConstTensor& values, + const typename TTypes::ConstTensor& value_range, + int32 nbins, typename TTypes::Tensor& out) { + tensorflow::AllocatorAttributes pinned_allocator; + pinned_allocator.set_on_host(true); + pinned_allocator.set_gpu_compatible(true); + + Tensor levels_tensor; + TF_RETURN_IF_ERROR(context->allocate_temp( + DataTypeToEnum::value, TensorShape({nbins + 1}), &levels_tensor, + pinned_allocator)); + auto levels = levels_tensor.flat(); + + const double step = static_cast(value_range(1) - value_range(0)) / + static_cast(nbins); + levels(0) = std::numeric_limits::lowest(); + for (int i = 1; i < nbins; i++) { + levels(i) = + static_cast(static_cast(value_range(0)) + step * i); + } + levels(nbins) = std::numeric_limits::max(); + + size_t temp_storage_bytes = 0; + const T* d_samples = values.data(); + Tout* d_histogram = out.data(); + int num_levels = levels.size(); + T* d_levels = levels.data(); + int num_samples = values.size(); + const cudaStream_t& stream = GetCudaStream(context); + + // The first HistogramRange is to obtain the temp storage size required + // with d_temp_storage = NULL passed to the call. + auto err = cub::DeviceHistogram::HistogramRange( + /* d_temp_storage */ NULL, + /* temp_storage_bytes */ temp_storage_bytes, + /* d_samples */ d_samples, + /* d_histogram */ d_histogram, + /* num_levels */ num_levels, + /* d_levels */ d_levels, + /* num_samples */ num_samples, + /* stream */ stream); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch HistogramRange to get temp storage: ", + cudaGetErrorString(err), "."); + } + + Tensor temp_storage; + TF_RETURN_IF_ERROR(context->allocate_temp( + DataTypeToEnum::value, + TensorShape({static_cast(temp_storage_bytes)}), &temp_storage)); + + void* d_temp_storage = temp_storage.flat().data(); + + // The second HistogramRange is to actual run with d_temp_storage + // allocated with temp_storage_bytes. + err = cub::DeviceHistogram::HistogramRange( + /* d_temp_storage */ d_temp_storage, + /* temp_storage_bytes */ temp_storage_bytes, + /* d_samples */ d_samples, + /* d_histogram */ d_histogram, + /* num_levels */ num_levels, + /* d_levels */ d_levels, + /* num_samples */ num_samples, + /* stream */ stream); + if (err != cudaSuccess) { + return errors::Internal("Could not launch HistogramRange: ", + cudaGetErrorString(err), "."); + } + + return Status::OK(); + } +}; + +} // end namespace functor + +#define REGISTER_GPU_SPEC(type) \ + template struct functor::HistogramFixedWidthFunctor; + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_SPEC); +#undef REGISTER_GPU_SPEC + +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 7b971a9fd5..a1c608ee54 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -2250,6 +2250,44 @@ product: Pairwise cross product of the vectors in `a` and `b`. // -------------------------------------------------------------------------- +REGISTER_OP("HistogramFixedWidth") + .Input("values: T") + .Input("value_range: T") + .Output("out: dtype") + .Attr("nbins: int = 100") + .Attr("T: {int32, int64, float32, float64}") + .Attr("dtype: {int32, int64} = DT_INT32") + .SetShapeFn([](InferenceContext* c) { + c->set_output(0, c->UnknownShapeOfRank(1)); + return Status::OK(); + }) + .Doc(R"doc( +Return histogram of values. + +Given the tensor `values`, this operation returns a rank 1 histogram counting +the number of entries in `values` that fall into every bin. The bins are +equal width and determined by the arguments `value_range` and `nbins`. + +```python +# Bins will be: (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf) +nbins = 5 +value_range = [0.0, 5.0] +new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] + +with tf.get_default_session() as sess: + hist = tf.histogram_fixed_width(new_values, value_range, nbins=5) + variables.global_variables_initializer().run() + sess.run(hist) => [2, 1, 1, 0, 2] +``` + +values: Numeric `Tensor`. +value_range: Shape [2] `Tensor` of same `dtype` as `values`. + values <= value_range[0] will be mapped to hist[0], + values >= value_range[1] will be mapped to hist[-1]. +nbins: Scalar `int32 Tensor`. Number of histogram bins. +out: A 1-D `Tensor` holding histogram of values. +)doc"); + REGISTER_OP("Bincount") .Input("arr: int32") .Input("size: int32") diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py index c2077d51af..040c3a5ae8 100644 --- a/tensorflow/python/ops/histogram_ops.py +++ b/tensorflow/python/ops/histogram_ops.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops @@ -69,30 +70,6 @@ def histogram_fixed_width(values, ``` """ with ops.name_scope(name, 'histogram_fixed_width', - [values, value_range, nbins]) as scope: - values = ops.convert_to_tensor(values, name='values') - values = array_ops.reshape(values, [-1]) - value_range = ops.convert_to_tensor(value_range, name='value_range') - nbins = ops.convert_to_tensor(nbins, dtype=dtypes.int32, name='nbins') - nbins_float = math_ops.cast(nbins, values.dtype) - - # Map tensor values that fall within value_range to [0, 1]. - scaled_values = math_ops.truediv(values - value_range[0], - value_range[1] - value_range[0], - name='scaled_values') - - # map tensor values within the open interval value_range to {0,.., nbins-1}, - # values outside the open interval will be zero or less, or nbins or more. - indices = math_ops.floor(nbins_float * scaled_values, name='indices') - - # Clip edge cases (e.g. value = value_range[1]) or "outliers." - indices = math_ops.cast( - clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32) - - # TODO(langmore) This creates an array of ones to add up and place in the - # bins. This is inefficient, so replace when a better Op is available. - return math_ops.unsorted_segment_sum( - array_ops.ones_like(indices, dtype=dtype), - indices, - nbins, - name=scope) + [values, value_range, nbins]) as name: + return gen_math_ops.histogram_fixed_width(values, value_range, nbins, + dtype=dtype, name=name) diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index e819e0234d..bf6e0296f6 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -36,7 +36,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [] expected_bin_counts = [0, 0, 0, 0, 0] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) @@ -47,7 +47,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width( values, value_range, nbins=5, dtype=dtypes.int64) self.assertEqual(dtypes.int64, hist.dtype) @@ -59,7 +59,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = np.float64([0.0, 5.0]) values = np.float64([-1.0, 0.0, 1.5, 2.0, 5.0, 15]) expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) @@ -70,7 +70,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]] expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) -- GitLab From 684f88fa7e61721c3264dc70abeed2b3e6fa7717 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Mon, 16 Oct 2017 17:22:26 -0700 Subject: [PATCH 049/573] [XLA:GPU] Don't crash with --vmodule=gpu_compiler=2 if we can't run ptxas. At --vmodule=gpu_compiler=2, we run ptxas over our generated PTX, to validate it, and also to dump out stats like the number of registers used. But previously, this would fail if your GPU was anything other than sm_35 (i.e. K20/40/80), because we didn't pass down cc_major/cc_minor to ptxas. And moreover, if ptxas failed to compile your program, we'd LOG(FATAL), which is probably no what you want. This change fixes both those issues. Tested on my local GTX1080. PiperOrigin-RevId: 172403304 --- .../compiler/xla/service/gpu/gpu_compiler.cc | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 57f11db11f..3e16e4e3c4 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -67,6 +67,7 @@ limitations under the License. #include "tensorflow/compiler/xla/util.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/cuda_libdevice_path.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" @@ -84,6 +85,8 @@ namespace gpu { namespace { +using tensorflow::strings::StrCat; + // Any address of a variable residing in global memory or returned by one of the // memory allocation routines from the driver or runtime API is always aligned // to at least 256 bytes. @@ -223,7 +226,7 @@ tensorflow::Status PrepareHloModuleForIrEmitting( } // Invokes the ptxas tool on the given PTX string, and dumps its output. -void DumpPtxasInfo(const string& ptx) { +void DumpPtxasInfo(const string& ptx, int cc_major, int cc_minor) { const string ptxas_path = tensorflow::io::JoinPath(tensorflow::CudaRoot(), "bin/ptxas"); // Do not log PTX stats if ptxas is not found at the given path. @@ -245,17 +248,22 @@ void DumpPtxasInfo(const string& ptx) { // Invoke ptxas and collect its output. tensorflow::SubProcess ptxas_info_dumper; - ptxas_info_dumper.SetProgram(ptxas_path, {ptxas_path, ptx_path, "-o", - "/dev/null", "-v", "-arch=sm_35"}); + ptxas_info_dumper.SetProgram(ptxas_path, + {ptxas_path, ptx_path, "-o", "/dev/null", "-v", + StrCat("-arch=sm_", cc_major, cc_minor)}); ptxas_info_dumper.SetChannelAction(tensorflow::CHAN_STDERR, tensorflow::ACTION_PIPE); - CHECK(ptxas_info_dumper.Start()); + if (!ptxas_info_dumper.Start()) { + LOG(ERROR) << "Failed to launch ptxas."; + return; + } string stderr_output; int exit_status = ptxas_info_dumper.Communicate( /*stdin_input=*/nullptr, /*stdout_output=*/nullptr, &stderr_output); XLA_LOG_LINES(tensorflow::INFO, stderr_output); if (exit_status != 0) { - LOG(FATAL) << "Invalid PTX. See the error message above for reasons."; + LOG(ERROR) << "ptxas exited with non-zero error code " << exit_status + << "."; } } @@ -387,7 +395,7 @@ StatusOr> GpuCompiler::Compile( VLOG(2) << "PTX:"; XLA_VLOG_LINES(2, *ptx); if (VLOG_IS_ON(2)) { - DumpPtxasInfo(*ptx); + DumpPtxasInfo(*ptx, cc_major, cc_minor); } auto thunk_schedule = MakeUnique( -- GitLab From 5c5dc8d5641b7c915f681109921dfb2b3e082a9b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 17:59:11 -0700 Subject: [PATCH 050/573] Adding an ItemHandler that does lookups. This allows decoding of tf.Examples where IDs are not materialized (e.g. 'image/object/class/text' present but 'image/object/class/label' not). PiperOrigin-RevId: 172406978 --- .../python/slim/data/tfexample_decoder.py | 36 +++++++++++++++++++ .../slim/data/tfexample_decoder_test.py | 31 ++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py index 094568389c..7a56df9e97 100644 --- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py +++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py @@ -207,6 +207,42 @@ class Tensor(ItemHandler): return tensor +class LookupTensor(Tensor): + """An ItemHandler that returns a parsed Tensor, the result of a lookup.""" + + def __init__(self, + tensor_key, + table, + shape_keys=None, + shape=None, + default_value=''): + """Initializes the LookupTensor handler. + + See Tensor. Simply calls a vocabulary (most often, a label mapping) lookup. + + Args: + tensor_key: the name of the `TFExample` feature to read the tensor from. + table: A tf.lookup table. + shape_keys: Optional name or list of names of the TF-Example feature in + which the tensor shape is stored. If a list, then each corresponds to + one dimension of the shape. + shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is + reshaped accordingly. + default_value: The value used when the `tensor_key` is not found in a + particular `TFExample`. + + Raises: + ValueError: if both `shape_keys` and `shape` are specified. + """ + self._table = table + super(LookupTensor, self).__init__(tensor_key, shape_keys, shape, + default_value) + + def tensors_to_item(self, keys_to_tensors): + unmapped_tensor = super(LookupTensor, self).tensors_to_item(keys_to_tensors) + return self._table.lookup(unmapped_tensor) + + class SparseTensor(ItemHandler): """An ItemHandler for SparseTensors.""" diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py index 60d1eba07f..9c5a14d006 100644 --- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py +++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import image_ops +from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test @@ -811,6 +812,36 @@ class TFExampleDecoderTest(test.TestCase): self.assertAllEqual(np.squeeze(output_image[0, :, :, :]), image) self.assertAllEqual(np.squeeze(output_image[1, :, :, :]), image) + def testDecodeExampleWithLookup(self): + + example = example_pb2.Example(features=feature_pb2.Features(feature={ + 'image/object/class/text': self._BytesFeature( + np.array(['cat', 'dog', 'guinea pig'])), + })) + serialized_example = example.SerializeToString() + # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2 + table = lookup_ops.index_table_from_tensor( + constant_op.constant(['dog', 'guinea pig', 'cat'])) + + with self.test_session() as sess: + sess.run(lookup_ops.tables_initializer()) + + serialized_example = array_ops.reshape(serialized_example, shape=[]) + + keys_to_features = { + 'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string), + } + + items_to_handlers = { + 'labels': + tfexample_decoder.LookupTensor('image/object/class/text', table), + } + + decoder = tfexample_decoder.TFExampleDecoder(keys_to_features, + items_to_handlers) + obtained_class_ids = decoder.decode(serialized_example)[0].eval() + + self.assertAllClose([2, 0, 1], obtained_class_ids) if __name__ == '__main__': test.main() -- GitLab From ecaa2eee832bd5b4286377f0f853c961c6ac2ab2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 18:06:20 -0700 Subject: [PATCH 051/573] math_grad: Fast path for when broadcasting is not needed. PiperOrigin-RevId: 172407754 --- tensorflow/contrib/compiler/jit_test.py | 28 ++++++------ .../graph_editor/tests/transform_test.py | 4 +- .../layers/python/layers/optimizers_test.py | 4 +- .../keras/_impl/keras/optimizers_test.py | 5 ++- tensorflow/python/ops/math_grad.py | 22 +++++++++- tensorflow/python/ops/rnn_cell_impl.py | 44 ++++++++++++++----- 6 files changed, 77 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/compiler/jit_test.py b/tensorflow/contrib/compiler/jit_test.py index 94aff13a49..2108e42bce 100644 --- a/tensorflow/contrib/compiler/jit_test.py +++ b/tensorflow/contrib/compiler/jit_test.py @@ -173,12 +173,12 @@ class CompilationEnabledInGradientTest(test.TestCase): def testCompilationInGradient(self): with self.test_session(): - x = constant_op.constant(3) - y_nc = math_ops.add(x, x, name="not_compiled") + x = constant_op.constant([[3]]) + y_nc = math_ops.matmul(x, x, name="not_compiled") with jit.experimental_jit_scope(): - y_c = math_ops.add(y_nc, y_nc, name="compiled") + y_c = math_ops.matmul(y_nc, y_nc, name="compiled") x_grads = gradients.gradients([y_c], [x])[0] - operations = x_grads.graph.get_operations() + operations = x.graph.get_operations() c_grad_ops = [ op for op in operations if "gradients/compiled" in op.name] nc_grad_ops = [ @@ -191,19 +191,19 @@ class CompilationEnabledInGradientTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "No attr named"): ncg.get_attr("_XlaCompile") - # d/dx (4 * x) - self.assertAllClose(4, x_grads.eval()) + # d/dx (x ** 4) = 4 * (x ** 3) + self.assertAllClose([[108]], x_grads.eval()) def testCompilationGradientScopeNames(self): with self.test_session(graph=ops.Graph()): with jit.experimental_jit_scope(): # XlaScope 0 - a1 = constant_op.constant(1) - a1t = a1 + a1 + a1 = constant_op.constant([[1]]) + a1t = math_ops.matmul(a1, a1) with jit.experimental_jit_scope(): # XlaScope 1 - a2 = constant_op.constant(1) - a2t = a2 + a2 + a2 = constant_op.constant([[1]]) + a2t = math_ops.matmul(a2, a2) self.assertEqual(b"jit_scope_0", a1.op.get_attr("_XlaScope")) self.assertEqual(b"jit_scope_1", a2.op.get_attr("_XlaScope")) @@ -220,12 +220,12 @@ class CompilationEnabledInGradientTest(test.TestCase): with self.test_session(graph=ops.Graph()): with jit.experimental_jit_scope(True, separate_compiled_gradients=True): # XlaScope 0 - a1 = constant_op.constant(1) - a1t = a1 + a1 + a1 = constant_op.constant([[1]]) + a1t = math_ops.matmul(a1, a1) with jit.experimental_jit_scope(True, separate_compiled_gradients=True): # XlaScope 1 - a2 = constant_op.constant(1) - a2t = a2 + a2 + a2 = constant_op.constant([[1]]) + a2t = math_ops.matmul(a2, a2) self.assertEqual(b"jit_scope_0", a1.op.get_attr("_XlaScope")) self.assertEqual(b"jit_scope_1", a2.op.get_attr("_XlaScope")) diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py index ab5776b9dd..ca00394388 100644 --- a/tensorflow/contrib/graph_editor/tests/transform_test.py +++ b/tensorflow/contrib/graph_editor/tests/transform_test.py @@ -191,14 +191,14 @@ class TransformTest(test.TestCase): # Extract the operations. replacement_ts = {w.value(): g} original_mul1_grad = (ops.get_default_graph(). - get_operation_by_name("grad/mul1_grad/mul_1")) + get_operation_by_name("grad/mul1_grad/Mul_1")) # Should not raise exception. res = ge.graph_replace(g, replacement_ts, dst_scope="res") # Extract the operations after graph_replace. result_mul1_grad = (ops.get_default_graph(). - get_operation_by_name("res/grad/mul1_grad/mul_1")) + get_operation_by_name("res/grad/mul1_grad/Mul_1")) # Make sure _original_ops are as expected. self.assertEquals(original_mul1_grad._original_op.name, u"mul1") diff --git a/tensorflow/contrib/layers/python/layers/optimizers_test.py b/tensorflow/contrib/layers/python/layers/optimizers_test.py index 8813a99f19..1ea25bd1a5 100644 --- a/tensorflow/contrib/layers/python/layers/optimizers_test.py +++ b/tensorflow/contrib/layers/python/layers/optimizers_test.py @@ -176,7 +176,7 @@ class OptimizersTest(test.TestCase): session.run(train, feed_dict={x: 5}) var_value, global_step_value = session.run([var, global_step]) # Due to randomness the following number may change if graph is different. - self.assertAlmostEqual(var_value, 8.5591021, 4) + self.assertAlmostEqual(var_value, 9.86912, 4) self.assertEqual(global_step_value, 1) def testGradientNoiseWithClipping(self): @@ -193,7 +193,7 @@ class OptimizersTest(test.TestCase): variables.global_variables_initializer().run() session.run(train, feed_dict={x: 5}) var_value, global_step_value = session.run([var, global_step]) - self.assertAlmostEqual(var_value, 9.0, 4) + self.assertAlmostEqual(var_value, 9.86912, 4) self.assertEqual(global_step_value, 1) def testGradientClip(self): diff --git a/tensorflow/python/keras/_impl/keras/optimizers_test.py b/tensorflow/python/keras/_impl/keras/optimizers_test.py index b63d82f6a0..6e9e4e6c99 100644 --- a/tensorflow/python/keras/_impl/keras/optimizers_test.py +++ b/tensorflow/python/keras/_impl/keras/optimizers_test.py @@ -93,7 +93,10 @@ class KerasOptimizersTest(test.TestCase): def test_adadelta(self): with self.test_session(): _test_optimizer(keras.optimizers.Adadelta(), target=0.6) - _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.6) + # Accuracy seems dependent on the initialization. Even adding tf.Print + # nodes in the graph seemed to affect the initialization seed, and hence + # the accuracy. + _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4) def test_adam(self): with self.test_session(): diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py index 3754e039ed..38fe093ba7 100644 --- a/tensorflow/python/ops/math_grad.py +++ b/tensorflow/python/ops/math_grad.py @@ -700,10 +700,26 @@ def _AddNGrad(op, grad): return [grad] * len(op.inputs) +def _ShapesFullySpecifiedAndEqual(x, y, grad): + # pylint: disable=protected-access + x_shape = x._shape_tuple() + y_shape = y._shape_tuple() + grad_shape = grad._shape_tuple() + # pylint: enable=protected-access + return (x_shape == y_shape and + x_shape == grad_shape and + x_shape is not None and + None not in x_shape) + + @ops.RegisterGradient("Add") def _AddGrad(op, grad): + """Gradient for Add.""" x = op.inputs[0] y = op.inputs[1] + if (isinstance(grad, ops.Tensor) and + _ShapesFullySpecifiedAndEqual(x, y, grad)): + return grad, grad sx = array_ops.shape(x) sy = array_ops.shape(y) # pylint: disable=protected-access @@ -731,10 +747,14 @@ def _MulGrad(op, grad): """The gradient of scalar multiplication.""" x = op.inputs[0] y = op.inputs[1] + # pylint: disable=protected-access + if (isinstance(grad, ops.Tensor) and + _ShapesFullySpecifiedAndEqual(x, y, grad) and + grad.dtype in (dtypes.int32, dtypes.float32)): + return gen_math_ops._mul(grad, y), gen_math_ops._mul(grad, x) assert x.dtype.base_dtype == y.dtype.base_dtype, (x.dtype, " vs. ", y.dtype) sx = array_ops.shape(x) sy = array_ops.shape(y) - # pylint: disable=protected-access rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy) # pylint: enable=protected-access x = math_ops.conj(x) diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index 4056eade81..fb7b6d11a5 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -178,8 +178,13 @@ class RNNCell(base_layer.Layer): custom_getter=self._rnn_get_variable) as scope: return super(RNNCell, self).__call__(inputs, state, scope=scope) else: - with vs.variable_scope(vs.get_variable_scope(), - custom_getter=self._rnn_get_variable): + scope_attrname = "rnncell_scope" + scope = getattr(self, scope_attrname, None) + if scope is None: + scope = vs.variable_scope(vs.get_variable_scope(), + custom_getter=self._rnn_get_variable) + setattr(self, scope_attrname, scope) + with scope: return super(RNNCell, self).__call__(inputs, state) def _rnn_get_variable(self, getter, *args, **kwargs): @@ -230,9 +235,20 @@ class RNNCell(base_layer.Layer): a nested list or tuple (of the same structure) of `2-D` tensors with the shapes `[batch_size x s]` for each s in `state_size`. """ + # Try to use the last cached zero_state. This is done to avoid recreating + # zeros, especially when eager execution is enabled. + state_size = self.state_size + if hasattr(self, "_last_zero_state"): + (last_state_size, last_batch_size, last_dtype, + last_output) = getattr(self, "_last_zero_state") + if (last_batch_size == batch_size and + last_dtype == dtype and + last_state_size == state_size): + return last_output with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - state_size = self.state_size - return _zero_state_tensors(state_size, batch_size, dtype) + output = _zero_state_tensors(state_size, batch_size, dtype) + self._last_zero_state = (state_size, batch_size, dtype, output) + return output class BasicRNNCell(RNNCell): @@ -428,21 +444,27 @@ class BasicLSTMCell(RNNCell): `state_is_tuple`). """ sigmoid = math_ops.sigmoid + one = constant_op.constant(1, dtype=dtypes.int32) # Parameters of gates are concatenated into one multiply for efficiency. if self._state_is_tuple: c, h = state else: - c, h = array_ops.split(value=state, num_or_size_splits=2, axis=1) + c, h = array_ops.split(value=state, num_or_size_splits=2, axis=one) if self._linear is None: self._linear = _Linear([inputs, h], 4 * self._num_units, True) # i = input_gate, j = new_input, f = forget_gate, o = output_gate i, j, f, o = array_ops.split( - value=self._linear([inputs, h]), num_or_size_splits=4, axis=1) + value=self._linear([inputs, h]), num_or_size_splits=4, axis=one) - new_c = ( - c * sigmoid(f + self._forget_bias) + sigmoid(i) * self._activation(j)) - new_h = self._activation(new_c) * sigmoid(o) + forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype) + # Note that using `add` and `multiply` instead of `+` and `*` gives a + # performance improvement. So using those at the cost of readability. + add = math_ops.add + multiply = math_ops.multiply + new_c = add(multiply(c, sigmoid(add(f, forget_bias_tensor))), + multiply(sigmoid(i), self._activation(j))) + new_h = multiply(self._activation(new_c), sigmoid(o)) if self._state_is_tuple: new_state = LSTMStateTuple(new_c, new_h) @@ -1186,7 +1208,9 @@ class _Linear(object): if len(args) == 1: res = math_ops.matmul(args[0], self._weights) else: - res = math_ops.matmul(array_ops.concat(args, 1), self._weights) + # Explicitly creating a one for a minor performance improvement. + one = constant_op.constant(1, dtype=dtypes.int32) + res = math_ops.matmul(array_ops.concat(args, one), self._weights) if self._build_bias: res = nn_ops.bias_add(res, self._biases) return res -- GitLab From a9da1baf65b42de9751959cccf6d899c69c0156b Mon Sep 17 00:00:00 2001 From: Vijay Vasudevan Date: Mon, 16 Oct 2017 18:19:58 -0700 Subject: [PATCH 052/573] Disable probable timeout flake on Ubuntu machines. PiperOrigin-RevId: 172408922 --- .../python/kernel_tests/dataset_from_generator_op_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py index e774256695..cd2bec8432 100644 --- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py +++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py @@ -135,7 +135,8 @@ class DatasetConstructorTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) - def testFromGeneratorsRunningInParallel(self): + # TODO(b/67868766): Reenable this when the source of flakiness is discovered. + def _testFromGeneratorsRunningInParallel(self): num_parallel_iterators = 3 # Define shared state that multiple iterator instances will access to -- GitLab From 0a092298823d73d1e4cc76e81e0825a8789cd6a5 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 16 Oct 2017 19:03:32 -0700 Subject: [PATCH 053/573] Make Snappy header available This is going to be useful for the tensor database I'm working on. PiperOrigin-RevId: 172412142 --- tensorflow/core/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 4d9f368bc0..94ddd0840d 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1398,6 +1398,7 @@ LIB_INTERNAL_PUBLIC_HEADERS = tf_additional_lib_hdrs() + [ "platform/platform.h", "platform/protobuf_internal.h", "platform/setround.h", + "platform/snappy.h", "platform/tensor_coding.h", "platform/tracing.h", ] @@ -2257,7 +2258,6 @@ cc_library( "lib/io/block_builder.h", "lib/io/format.h", "lib/random/philox_random_test_utils.h", - "platform/snappy.h", ], deps = [ ":lib", -- GitLab From ba5a5bfc23065086990ec3057caa2ded0c8a8dbf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Oct 2017 21:59:25 -0700 Subject: [PATCH 054/573] Add the op->IsExpensive() argument to tracing calls. PiperOrigin-RevId: 172422580 --- tensorflow/compiler/jit/xla_device.cc | 6 +++-- .../core/common_runtime/gpu/gpu_device.cc | 3 ++- .../core/common_runtime/threadpool_device.cc | 3 ++- .../core/platform/default/gpu_tracer.cc | 5 +++- tensorflow/core/platform/tracing.h | 25 ++++++++++++------- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index 7ccea58f6e..d4d8fe1c1d 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -241,7 +241,8 @@ void XlaDevice::Compute(OpKernel* op_kernel, OpKernelContext* context) { // When TraceMe profiling is off (which is the default), the // following TraceMe constructor is simply a conditional test of // false value. Measurements show that its overhead is negligible. - port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); op_kernel->Compute(context); } @@ -249,7 +250,8 @@ void XlaDevice::ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context, AsyncOpKernel::DoneCallback done) { VLOG(1) << "XlaDevice::ComputeAsync " << op_kernel->name() << ":" << op_kernel->type_string(); - port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); op_kernel->ComputeAsync(context, done); } diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc index 3324e833ff..12d44cc6b7 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_device.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc @@ -475,7 +475,8 @@ void BaseGPUDevice::ComputeAsync(AsyncOpKernel* op_kernel, // When TraceMe profiling is off (which is the default), the // following TraceMe constructor is simply a conditional test of // false value. Measurements show that its overhead is negligible. - port::Tracing::TraceMe activity(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe activity(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); gpu::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()}; op_kernel->ComputeAsync(context, done); } diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc index 23ccca1c94..5aa01376ab 100644 --- a/tensorflow/core/common_runtime/threadpool_device.cc +++ b/tensorflow/core/common_runtime/threadpool_device.cc @@ -48,7 +48,8 @@ void ThreadPoolDevice::Compute(OpKernel* op_kernel, OpKernelContext* context) { // When TraceMe profiling is off (which is the default), the // following TraceMe constructor is simply a conditional test of // false value. Measurements show that its overhead is negligible. - port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); if (port::Tracing::IsActive()) { // TODO(pbar) We really need a useful identifier of the graph node. const uint64 id = Hash64(op_kernel->name()); diff --git a/tensorflow/core/platform/default/gpu_tracer.cc b/tensorflow/core/platform/default/gpu_tracer.cc index 3f85546127..e52e37ad71 100644 --- a/tensorflow/core/platform/default/gpu_tracer.cc +++ b/tensorflow/core/platform/default/gpu_tracer.cc @@ -315,10 +315,13 @@ class GPUTracerImpl : public GPUTracer, }; return new Impl(name); } - Tracer *StartTracing(StringPiece label) override { + Tracer *StartTracing(StringPiece label, bool is_expensive) override { // We don't do anything with 'TraceMe' regions yet. return nullptr; } + Tracer *StartTracing(StringPiece label) { + return StartTracing(label, /*is_expensive=*/true); + } protected: // This callback is used exclusively by CUPTIManager. diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h index b7724bbeae..bb8e902efc 100644 --- a/tensorflow/core/platform/tracing.h +++ b/tensorflow/core/platform/tracing.h @@ -169,10 +169,10 @@ class Tracing::Engine { // Start tracing under the specified label. Caller should delete the result // to stop tracing. // May return nullptr if tracing is not supported. - virtual Tracer* StartTracing(StringPiece label) = 0; + virtual Tracer* StartTracing(StringPiece label, bool is_expensive) = 0; // Same as above, but implementations can avoid copying the string. - virtual Tracer* StartTracing(string&& label) { - return StartTracing(StringPiece(label)); + virtual Tracer* StartTracing(string&& label, bool is_expensive) { + return StartTracing(StringPiece(label), is_expensive); } }; @@ -218,12 +218,14 @@ class Tracing::ScopedAnnotation { class Tracing::TraceMe { public: explicit TraceMe(StringPiece name); + TraceMe(StringPiece name, bool is_expensive); // If tracing is enabled, set up a traceMe with a label of // ":". This can be cheaper than the // single-argument constructor because the concatenation of the // label string is only done if tracing is enabled. TraceMe(StringPiece name_part1, StringPiece name_part2); + TraceMe(StringPiece name_part1, StringPiece name_part2, bool is_expensive); private: std::unique_ptr tracer_; @@ -245,19 +247,24 @@ inline Tracing::ScopedAnnotation::ScopedAnnotation(StringPiece name_part1, } } -inline Tracing::TraceMe::TraceMe(StringPiece name) { +inline Tracing::TraceMe::TraceMe(StringPiece name) : TraceMe(name, true) {} + +inline Tracing::TraceMe::TraceMe(StringPiece name, bool is_expensive) { auto e = Tracing::engine(); if (e && e->IsEnabled()) { - tracer_.reset(e->StartTracing(name)); + tracer_.reset(e->StartTracing(name, is_expensive)); } } -inline Tracing::TraceMe::TraceMe(StringPiece name_part1, - StringPiece name_part2) { +inline Tracing::TraceMe::TraceMe(StringPiece name_part1, StringPiece name_part2) + : TraceMe(name_part1, name_part2, true) {} + +inline Tracing::TraceMe::TraceMe(StringPiece name_part1, StringPiece name_part2, + bool is_expensive) { auto e = Tracing::engine(); if (e && e->IsEnabled()) { - tracer_.reset( - e->StartTracing(strings::StrCat(name_part1, ":", name_part2))); + tracer_.reset(e->StartTracing(strings::StrCat(name_part1, ":", name_part2), + is_expensive)); } } -- GitLab From a1ba9f3bf16cb53b8468b93021611311a9be55b4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 08:46:12 -0700 Subject: [PATCH 055/573] HParams: deprecates function set_from_map in favor of new function override_from_dict Reasons to prefer new function name: - `set` sounds like it might return the builtin set. - There is no datatype `map` in python - it's a builtin, making the implied API a little confusing. PiperOrigin-RevId: 172471191 --- .../contrib/training/python/training/hparam.py | 18 ++++++++++++------ .../training/python/training/hparam_test.py | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py index c95a73ce44..1b52d23c61 100644 --- a/tensorflow/contrib/training/python/training/hparam.py +++ b/tensorflow/contrib/training/python/training/hparam.py @@ -25,6 +25,7 @@ import six from tensorflow.contrib.training.python.training import hparam_pb2 from tensorflow.python.framework import ops from tensorflow.python.util import compat +from tensorflow.python.util import deprecation # Define the regular expression for parsing a single clause of the input # (delimited by commas). A legal clause looks like: @@ -470,24 +471,29 @@ class HParams(object): type_map[name] = param_type values_map = parse_values(values, type_map) - return self.set_from_map(values_map) + return self.override_from_dict(values_map) - def set_from_map(self, values_map): + def override_from_dict(self, values_dict): """Override hyperparameter values, parsing new values from a dictionary. Args: - values_map: Dictionary of name:value pairs. + values_dict: Dictionary of name:value pairs. Returns: The `HParams` instance. Raises: - ValueError: If `values_map` cannot be parsed. + ValueError: If `values_dict` cannot be parsed. """ - for name, value in values_map.items(): + for name, value in values_dict.items(): self.set_hparam(name, value) return self + @deprecation.deprecated(None, 'Use `override_from_dict`.') + def set_from_map(self, values_map): + """DEPRECATED. Use override_from_dict.""" + return self.override_from_dict(values_dict=values_map) + def set_model_structure(self, model_structure): self._model_structure = model_structure @@ -515,7 +521,7 @@ class HParams(object): ValueError: If `values_json` cannot be parsed. """ values_map = json.loads(values_json) - return self.set_from_map(values_map) + return self.override_from_dict(values_map) def values(self): """Return the hyperparameter values as a Python dictionary. diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py index b01116a213..a947bf6eda 100644 --- a/tensorflow/contrib/training/python/training/hparam_test.py +++ b/tensorflow/contrib/training/python/training/hparam_test.py @@ -93,11 +93,11 @@ class HParamsTest(test.TestCase): def testSetFromMap(self): hparams = hparam.HParams(a=1, b=2.0, c='tanh') - hparams.set_from_map({'a': -2, 'c': 'identity'}) + hparams.override_from_dict({'a': -2, 'c': 'identity'}) self.assertDictEqual({'a': -2, 'c': 'identity', 'b': 2.0}, hparams.values()) hparams = hparam.HParams(x=1, b=2.0, d=[0.5]) - hparams.set_from_map({'d': [0.1, 0.2, 0.3]}) + hparams.override_from_dict({'d': [0.1, 0.2, 0.3]}) self.assertDictEqual({'d': [0.1, 0.2, 0.3], 'x': 1, 'b': 2.0}, hparams.values()) -- GitLab From 18f89c81d288f191abd56501ec6f86fe29265bdd Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Tue, 17 Oct 2017 08:48:29 -0700 Subject: [PATCH 056/573] [tf.contrib.seq2seq] Bugfixes to BeamSearchDecoder and GatherTree. 1. Begin the gather tree at the maximum sequence length across all beams (within the batch). 2. Take a second pass starting from t=0 and mask out any beam ids past the *first* beam occurrence of end_token. 3. Update the final sequence lengths to include the first token in the beam. 4. Update dynamic_decode to allow the BeamSearchDecoder to keep track of its own "finished" states, as the shuffling in the decoder confused the tracking mechanism in dynamic_decode. This fixes a bug where beam search decoding stops early. 5. Cap sequence length used in GatherTree to min(max_time, max_seq_len(b)) to avoid accessing memory outside the dimensions of input matrices. Bugs caught by @bdaskalov on github and Pavel Sountsov. Proper solution and analysis thanks to Rui Zhao. Thanks all! Fixes #13536. PiperOrigin-RevId: 172471462 --- .../seq2seq/kernels/beam_search_ops.cc | 104 ++++++++------- .../contrib/seq2seq/kernels/beam_search_ops.h | 4 +- .../seq2seq/kernels/beam_search_ops_gpu.cu.cc | 32 +++-- .../contrib/seq2seq/ops/beam_search_ops.cc | 25 ++-- .../kernel_tests/beam_search_decoder_test.py | 9 +- .../kernel_tests/beam_search_ops_test.py | 118 ++++++++---------- .../seq2seq/python/ops/beam_search_decoder.py | 39 ++++-- .../contrib/seq2seq/python/ops/decoder.py | 33 ++++- 8 files changed, 217 insertions(+), 147 deletions(-) diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc index aab0f3f494..95273e2b33 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc @@ -49,40 +49,46 @@ class GatherTreeOp : public OpKernel { const Device& device = ctx->eigen_device(); const Tensor& step_ids = ctx->input(0); const Tensor& parent_ids = ctx->input(1); - const Tensor& sequence_length = ctx->input(2); + const Tensor& max_sequence_lengths = ctx->input(2); + const Tensor& end_token = ctx->input(3); const TensorShape& step_ids_shape = step_ids.shape(); OP_REQUIRES( ctx, step_ids_shape.dims() == 3, errors::InvalidArgument("step_ids must be a 3-tensor, saw shape: ", step_ids_shape.DebugString())); - OP_REQUIRES( - ctx, TensorShapeUtils::IsMatrix(sequence_length.shape()), - errors::InvalidArgument("sequence_length must be a matrix, saw shape: ", - sequence_length.shape().DebugString())); - OP_REQUIRES(ctx, sequence_length.dim_size(0) == step_ids_shape.dim_size(1), - errors::InvalidArgument( - "Inconsistent batch sizes: sequence_length.shape[0] (", - sequence_length.dim_size(0), ") != ", "step_ids.shape[1] (", - step_ids_shape.dim_size(1), ")")); - OP_REQUIRES(ctx, sequence_length.dim_size(1) == step_ids_shape.dim_size(2), + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(max_sequence_lengths.shape()), errors::InvalidArgument( - "Inconsistent batch sizes: sequence_length.shape[1] (", - sequence_length.dim_size(1), ") != ", "step_ids.shape[2] (", - step_ids_shape.dim_size(2), ")")); + "max_sequence_lengths must be a vector, saw shape: ", + max_sequence_lengths.shape().DebugString())); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(end_token.shape()), + errors::InvalidArgument("end_token must be a scalar, saw shape: ", + end_token.shape().DebugString())); OP_REQUIRES( ctx, step_ids_shape == parent_ids.shape(), errors::InvalidArgument( "step_ids.shape must match parent_ids.shape. but shapes are: ", step_ids_shape.DebugString(), " and ", parent_ids.shape().DebugString())); + OP_REQUIRES( + ctx, + step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0), + errors::InvalidArgument("batch size dimensions step_ids.shape[1] and " + "max_seqeuence_lengths.shape[0] must match. " + "but shapes are: ", + step_ids_shape.DebugString(), " and ", + max_sequence_lengths.shape().DebugString())); Tensor* beams; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, step_ids_shape, &beams)); typename TTypes::ConstTensor step_ids_t = step_ids.tensor(); typename TTypes::ConstTensor parent_ids_t = parent_ids.tensor(); - typename TTypes::ConstMatrix seq_len_t = sequence_length.matrix(); + typename TTypes::ConstVec max_seq_lens_t = + max_sequence_lengths.vec(); + typename TTypes::ConstScalar end_token_t = end_token.scalar(); typename TTypes::Tensor beams_t = beams->tensor(); + const T end_token_value = end_token_t(); functor::GatherTree()(ctx, device, step_ids_t, parent_ids_t, - seq_len_t, beams_t); + max_seq_lens_t, end_token_value, beams_t); } }; @@ -99,27 +105,29 @@ namespace functor { template <> struct GatherTree { void operator()(OpKernelContext* ctx, const CPUDevice& d, - typename TTypes::ConstTensor step_ids, - typename TTypes::ConstTensor parent_ids, - typename TTypes::ConstMatrix sequence_length, - typename TTypes::Tensor beams) { - const int64 max_time = parent_ids.dimension(0); - const int64 batch_size = parent_ids.dimension(1); - const int64 beam_width = parent_ids.dimension(2); + TTypes::ConstTensor step_ids, + TTypes::ConstTensor parent_ids, + TTypes::ConstVec max_sequence_lengths, + const int32 end_token, TTypes::Tensor beams) { + const int32 max_time = parent_ids.dimension(0); + const int32 batch_size = parent_ids.dimension(1); + const int32 beam_width = parent_ids.dimension(2); beams.setConstant(-1); - auto DoWork = [&, ctx](int start_batch_beam, int limit_batch_beam) { + auto DoWork = [&, ctx, end_token](int start_batch_beam, + int limit_batch_beam) { for (int32 i = start_batch_beam; i < limit_batch_beam; ++i) { const int32 batch = i / beam_width; const int32 beam = i % beam_width; - int32 seq_len_b = sequence_length(batch, beam); - if (seq_len_b <= 0) { + const int32 max_seq_len_b = + Eigen::numext::mini(max_time, max_sequence_lengths(batch)); + if (max_seq_len_b <= 0) { continue; } - beams(seq_len_b - 1, batch, beam) = - step_ids(seq_len_b - 1, batch, beam); - int32 parent = parent_ids(seq_len_b - 1, batch, beam); - for (int32 level = seq_len_b - 2; level >= 0; --level) { + beams(max_seq_len_b - 1, batch, beam) = + step_ids(max_seq_len_b - 1, batch, beam); + int32 parent = parent_ids(max_seq_len_b - 1, batch, beam); + for (int32 level = max_seq_len_b - 2; level >= 0; --level) { if (parent < 0 || parent > beam_width) { ctx->SetStatus( errors::InvalidArgument("Saw invalid parent id ", parent, @@ -130,6 +138,14 @@ struct GatherTree { beams(level, batch, beam) = step_ids(level, batch, parent); parent = parent_ids(level, batch, parent); } + bool finished = false; + for (int32 time = 0; time < max_seq_len_b; ++time) { + if (finished) { + beams(time, batch, beam) = -1; + } else if (beams(time, batch, beam) == end_token) { + finished = true; + } + } } }; // Guesstimate of cost; ~5 lookup/store/compare per inner beam @@ -137,7 +153,7 @@ struct GatherTree { const int64 batch_beam_cost = Eigen::TensorOpCost::DivCost() + 6 * Eigen::TensorOpCost::AddCost() + - max_time * (5 * Eigen::TensorOpCost::AddCost()); + 2 * max_time * (5 * Eigen::TensorOpCost::AddCost()); auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads()); Shard(worker_threads.num_threads, worker_threads.workers, batch_size * beam_width, batch_beam_cost, DoWork); @@ -148,24 +164,26 @@ struct GatherTree { #if GOOGLE_CUDA namespace functor { -#define DECLARE_GPU_SPEC(T) \ - template <> \ - void GatherTree::operator()( \ - OpKernelContext* ctx, const GPUDevice& d, \ - typename TTypes::ConstTensor step_ids, \ - typename TTypes::ConstTensor parent_ids, \ - typename TTypes::ConstMatrix sequence_length, \ - typename TTypes::Tensor beams); \ +#define DECLARE_GPU_SPEC(T) \ + template <> \ + void GatherTree::operator()( \ + OpKernelContext* ctx, const GPUDevice& d, \ + typename TTypes::ConstTensor step_ids, \ + typename TTypes::ConstTensor parent_ids, \ + TTypes::ConstVec max_sequence_lengths, const T end_token, \ + typename TTypes::Tensor beams); \ extern template struct GatherTree; DECLARE_GPU_SPEC(int32); #undef DECLARE_GPU_SPEC } // end namespace functor -#define REGISTER_GPU_KERNEL(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("GatherTree").Device(DEVICE_GPU).TypeConstraint("T"), \ - GatherTreeOp); +#define REGISTER_GPU_KERNEL(T) \ + REGISTER_KERNEL_BUILDER(Name("GatherTree") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("end_token"), \ + GatherTreeOp); REGISTER_GPU_KERNEL(int32); #undef REGISTER_GPU_KERNEL diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h index 124d07264e..693b02dc43 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h @@ -31,8 +31,8 @@ struct GatherTree { void operator()(OpKernelContext* ctx, const Device& d, typename TTypes::ConstTensor step_ids, typename TTypes::ConstTensor parent_ids, - typename TTypes::ConstMatrix sequence_length, - typename TTypes::Tensor beams); + TTypes::ConstVec max_sequence_lengths, + const T end_token, typename TTypes::Tensor beams); }; } // namespace functor diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc index ee68b55d20..e71efc48ce 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc @@ -29,20 +29,24 @@ template __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time, const int32 beam_width, const T* step_ids, const T* parent_ids, - const T* sequence_length, T* beams) { + const int32* max_sequence_lengths, + const T end_token, T* beams) { CUDA_1D_KERNEL_LOOP(i, batch_size * beam_width) { const int32 batch = i / beam_width; const int32 beam = i % beam_width; - const int32 seq_len_b = ldg(sequence_length + batch * beam_width + beam); - if (seq_len_b <= 0) continue; + const int32 max_seq_len_b = + Eigen::numext::mini(max_time, ldg(max_sequence_lengths + batch)); + if (max_seq_len_b <= 0) { + continue; + } #define GET_IX(time_ix, beam_ix) \ (batch_size * beam_width * (time_ix) + beam_width * batch + (beam_ix)) - const int32 initial_beam_ix = GET_IX(seq_len_b - 1, beam); + const int32 initial_beam_ix = GET_IX(max_seq_len_b - 1, beam); beams[initial_beam_ix] = ldg(step_ids + initial_beam_ix); int32 parent = ldg(parent_ids + initial_beam_ix); - for (int32 level = seq_len_b - 2; level >= 0; --level) { + for (int32 level = max_seq_len_b - 2; level >= 0; --level) { const int32 level_beam_ix = GET_IX(level, beam); const int32 level_parent_ix = GET_IX(level, parent); if (parent < 0 || parent > beam_width) { @@ -53,6 +57,15 @@ __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time, parent = ldg(parent_ids + level_parent_ix); } } + bool finished = false; + for (int32 time = 0; time < max_seq_len_b; ++time) { + const int32 level_beam_ix = GET_IX(time, beam); + if (finished) { + beams[level_beam_ix] = -1; + } else if (beams[level_beam_ix] == end_token) { + finished = true; + } + } #undef GET_IX } } @@ -62,8 +75,8 @@ struct GatherTree { void operator()(OpKernelContext* ctx, const GPUDevice& d, typename TTypes::ConstTensor step_ids, typename TTypes::ConstTensor parent_ids, - typename TTypes::ConstMatrix sequence_length, - typename TTypes::Tensor beams) { + TTypes::ConstVec max_sequence_length, + const T end_token, typename TTypes::Tensor beams) { const int32 max_time = parent_ids.dimension(0); const int32 batch_size = parent_ids.dimension(1); const int32 beam_width = parent_ids.dimension(2); @@ -75,7 +88,10 @@ struct GatherTree { GatherTreeOpKernel <<>>( batch_size, max_time, beam_width, - step_ids.data(), parent_ids.data(), sequence_length.data(), + step_ids.data(), + parent_ids.data(), + max_sequence_length.data(), + end_token, beams.data()); // clang-format on } diff --git a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc index 6c445cd460..231504bfbb 100644 --- a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc +++ b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc @@ -25,27 +25,27 @@ using shape_inference::ShapeHandle; REGISTER_OP("GatherTree") .Input("step_ids: T") .Input("parent_ids: T") - .Input("sequence_length: T") + .Input("max_sequence_lengths: int32") + .Input("end_token: T") .Output("beams: T") .Attr("T: {int32}") .SetShapeFn([](InferenceContext* c) { - ShapeHandle step_ids, parent_ids, sequence_length; + ShapeHandle step_ids, parent_ids, max_sequence_lengths, end_token; // step_ids, parent_ids, and output are all shaped: // [max_time, batch_size, beam_width]. - // sequence_length is shaped [batch_size, beam_width]. + // max_sequence_length is shaped [batch_size] and end_token is a scalar. TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 3, &step_ids)); TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &parent_ids)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &sequence_length)); - - DimensionHandle batch_size = c->Dim(step_ids, 1); - DimensionHandle beam_width = c->Dim(step_ids, 2); - + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &max_sequence_lengths)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &end_token)); TF_RETURN_IF_ERROR(c->Merge(step_ids, parent_ids, &step_ids)); + DimensionHandle batch_size = c->Dim(step_ids, 1); TF_RETURN_IF_ERROR( - c->Merge(batch_size, c->Dim(sequence_length, 0), &batch_size)); - TF_RETURN_IF_ERROR( - c->Merge(beam_width, c->Dim(sequence_length, 1), &beam_width)); + c->Merge(batch_size, c->Dim(max_sequence_lengths, 0), &batch_size)); + ShapeHandle step_ids_prefix = c->Matrix(c->Dim(step_ids, 0), batch_size); + TF_RETURN_IF_ERROR(c->MergePrefix(step_ids, step_ids_prefix, &step_ids, + &step_ids_prefix)); c->set_output(0, step_ids); return tensorflow::Status::OK(); @@ -61,7 +61,8 @@ TODO(ebrevdo): fill in step_ids: `[max_time, batch_size, beam_width]`. parent_ids: `[max_time, batch_size, beam_width]`. -sequence_length: `[batch_size, beam_width]`. +max_sequence_lengths: `[batch_size]`. +end_token: `[]`. beams: `[max_time, batch_size, beam_width]`. )doc"); diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py index 8d4ec4b4db..d2beac5f31 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py @@ -54,15 +54,18 @@ class TestGatherTree(test.TestCase): [[0, 0, 0], [1, 2, 0], [2, 1, 1]]], dtype=np.int32).transpose([1, 0, 2]) - # sequence_lengths is shaped (batch_size = 2, beam_width = 3) - sequence_lengths = [[3, 3, 3], [3, 3, 3]] + # sequence_lengths is shaped (batch_size = 3) + max_sequence_lengths = [3, 3] expected_result = np.array( [[[2, 2, 2], [6, 5, 6], [7, 8, 9]], [[2, 4, 4], [7, 6, 6], [8, 9, 10]]]).transpose([1, 0, 2]) res = beam_search_ops.gather_tree( - predicted_ids, parent_ids, sequence_lengths) + predicted_ids, + parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=11) with self.test_session() as sess: res_ = sess.run(res) diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py index 50cccf392f..f301314872 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py @@ -19,6 +19,8 @@ from __future__ import division from __future__ import print_function # pylint: enable=unused-import +import itertools + import numpy as np from tensorflow.contrib.seq2seq.python.ops import beam_search_ops @@ -38,12 +40,14 @@ class GatherTreeTest(test.TestCase): [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]]]) - sequence_length = [[3, 3, 3]] + max_sequence_lengths = [3] expected_result = _transpose_batch_time( [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]]) beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) + step_ids=step_ids, + parent_ids=parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=10) with self.test_session(use_gpu=True): self.assertAllEqual(expected_result, beams.eval()) @@ -54,11 +58,13 @@ class GatherTreeTest(test.TestCase): [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) - sequence_length = [[3, 3, 3]] + max_sequence_lengths = [3] with ops.device("/cpu:0"): beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) + step_ids=step_ids, + parent_ids=parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=10) with self.test_session(): with self.assertRaisesOpError( r"parent id -1 at \(batch, time, beam\) == \(0, 0, 1\)"): @@ -75,78 +81,58 @@ class GatherTreeTest(test.TestCase): [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) - sequence_length = [[3, 3, 3]] + max_sequence_lengths = [3] expected_result = _transpose_batch_time( [[[2, -1, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]]) with ops.device("/device:GPU:0"): beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) + step_ids=step_ids, + parent_ids=parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=10) with self.test_session(use_gpu=True): self.assertAllEqual(expected_result, beams.eval()) def testGatherTreeBatch(self): - # sequence_length is [batch_size, beam_width] = [4, 5] - sequence_length = [[0] * 5, [1] * 5, [2] * 5, [3] * 5] + batch_size = 10 + beam_width = 15 + max_time = 8 + max_sequence_lengths = [0, 1, 2, 4, 7, 8, 9, 10, 11, 0] + end_token = 5 with self.test_session(use_gpu=True): - # (max_time = 4, batch_size = 4, beam_width = 5) - step_ids = _transpose_batch_time( - [[[3, 4, 0, 4, 0], - [4, 2, 0, 3, 1], - [1, 1, 3, 2, 2], - [3, 1, 2, 3, 4]], - [[3, 4, 0, 4, 0], - [4, 2, 0, 3, 1], - [1, 1, 3, 2, 2], - [3, 1, 2, 3, 4]], - [[1, 2, 3, 4, 2], - [2, 1, 1, 3, 2], - [3, 0, 1, 0, 0], - [3, 4, 0, 2, 4]], - [[0, 2, 2, 3, 1], - [3, 2, 2, 2, 3], - [3, 4, 3, 0, 3], - [1, 2, 2, 2, 4]]]) - parent_ids = _transpose_batch_time( - [[[4, 2, 4, 3, 4], - [3, 4, 0, 2, 0], - [3, 1, 3, 2, 2], - [0, 2, 1, 4, 2]], - [[4, 2, 4, 3, 4], - [3, 4, 0, 2, 0], - [3, 1, 3, 2, 2], - [0, 2, 1, 4, 2]], - [[3, 0, 0, 4, 0], - [1, 2, 4, 2, 2], - [4, 4, 0, 3, 0], - [2, 4, 4, 3, 0]], - [[3, 1, 4, 1, 3], - [3, 2, 4, 0, 4], - [1, 0, 1, 4, 2], - [0, 3, 2, 0, 1]]]) - expected_beams = _transpose_batch_time( - [[[-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1]], - [[3, 4, 0, 4, 0], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1]], - [[2, 3, 2, 3, 3], - [2, 1, 1, 3, 2], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1]], - [[2, 3, 2, 1, 1], - [2, 3, 2, 3, 2], - [3, 4, 3, 0, 3], - [-1, -1, -1, -1, -1]]]) + step_ids = np.random.randint( + 0, high=end_token + 1, size=(max_time, batch_size, beam_width)) + parent_ids = np.random.randint( + 0, high=beam_width - 1, size=(max_time, batch_size, beam_width)) beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) - self.assertAllEqual(expected_beams, beams.eval()) + step_ids=step_ids.astype(np.int32), + parent_ids=parent_ids.astype(np.int32), + max_sequence_lengths=max_sequence_lengths, + end_token=end_token) + + self.assertEqual((max_time, batch_size, beam_width), beams.shape) + beams_value = beams.eval() + for b in range(batch_size): + # Past max_sequence_lengths[b], we emit all -1s. + b_value = beams_value[max_sequence_lengths[b]:, b, :] + self.assertAllClose(b_value, -1. * np.ones_like(b_value)) + for batch, beam in itertools.product( + range(batch_size), range(beam_width)): + v = np.squeeze(beams_value[:, batch, beam]) + if end_token in v: + found = np.where(v == end_token)[0] + # Should be up to 1 instance of end_token per beam. + self.assertEqual(len(found), 1) + found = found[0] + # If an end_token is found, everything before it should be a + # valid id and everything after it should be -1. + if found > 0: + self.assertAllEqual( + v[:found - 1] >= 0, np.ones_like(v[:found - 1], dtype=bool)) + self.assertAllClose( + v[found + 1:], -1 * np.ones_like(v[found + 1:])) if __name__ == "__main__": diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py index 112ac57a1b..a88d4f5b8b 100644 --- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py +++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py @@ -253,6 +253,20 @@ class BeamSearchDecoder(decoder.Decoder): output_shape_with_unknown_batch) return nest.map_structure(lambda s: s[1:], layer_output_shape) + @property + def tracks_own_finished(self): + """The BeamSearchDecoder shuffles its beams and their finished state. + + For this reason, it conflicts with the `dynamic_decode` function's + tracking of finished states. Setting this property to true avoids + early stopping of decoding due to mismanagement of the finished state + in `dynamic_decode`. + + Returns: + `True`. + """ + return True + @property def output_size(self): # Return the cell output and the id @@ -303,15 +317,23 @@ class BeamSearchDecoder(decoder.Decoder): output. sequence_lengths: An `int64` tensor shaped `[batch_size, beam_width]`. The sequence lengths determined for each beam during decode. + **NOTE** These are ignored; the updated sequence lengths are stored in + `final_state.lengths`. Returns: - outputs: An instance of FinalBeamSearchDecoderOutput where the + outputs: An instance of `FinalBeamSearchDecoderOutput` where the predicted_ids are the result of calling _gather_tree. - final_state: The same input instance of BeamSearchDecoderState. + final_state: The same input instance of `BeamSearchDecoderState`. """ + del sequence_lengths + # Get max_sequence_length across all beams for each batch. + max_sequence_lengths = math_ops.to_int32( + math_ops.reduce_max(final_state.lengths, axis=1)) predicted_ids = beam_search_ops.gather_tree( - outputs.predicted_ids, outputs.parent_ids, - sequence_length=sequence_lengths) + outputs.predicted_ids, + outputs.parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=self._end_token) outputs = FinalBeamSearchDecoderOutput( beam_search_decoder_output=outputs, predicted_ids=predicted_ids) return outputs, final_state @@ -588,10 +610,11 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size, name="next_beam_finished") # Calculate the length of the next predictions. - # 1. Finished beams remain unchanged - # 2. Beams that are now finished (EOS predicted) remain unchanged - # 3. Beams that are not yet finished have their length increased by 1 - lengths_to_add = math_ops.to_int64(math_ops.logical_not(next_finished)) + # 1. Finished beams remain unchanged. + # 2. Beams that are now finished (EOS predicted) have their length + # increased by 1. + # 3. Beams that are not yet finished have their length increased by 1. + lengths_to_add = math_ops.to_int64(math_ops.logical_not(previously_finished)) next_prediction_len = _tensor_gather_helper( gather_indices=next_beam_ids, gather_from=beam_state.lengths, diff --git a/tensorflow/contrib/seq2seq/python/ops/decoder.py b/tensorflow/contrib/seq2seq/python/ops/decoder.py index fbe53fc60a..f14974b9d5 100644 --- a/tensorflow/contrib/seq2seq/python/ops/decoder.py +++ b/tensorflow/contrib/seq2seq/python/ops/decoder.py @@ -100,16 +100,36 @@ class Decoder(object): Returns: `(outputs, next_state, next_inputs, finished)`: `outputs` is an object - containing the decoder output, `next_state` is a (structure of) state tensors - and TensorArrays, `next_inputs` is the tensor that should be used as input for - the next step, `finished` is a boolean tensor telling whether the sequence - is complete, for each sequence in the batch. + containing the decoder output, `next_state` is a (structure of) state + tensors and TensorArrays, `next_inputs` is the tensor that should be used + as input for the next step, `finished` is a boolean tensor telling whether + the sequence is complete, for each sequence in the batch. """ raise NotImplementedError def finalize(self, outputs, final_state, sequence_lengths): raise NotImplementedError + @property + def tracks_own_finished(self): + """Describes whether the Decoder keeps track of finished states. + + Most decoders will emit a true/false `finished` value independently + at each time step. In this case, the `dynamic_decode` function keeps track + of which batch entries are already finished, and performs a logical OR to + insert new batches to the finished set. + + Some decoders, however, shuffle batches / beams between time steps and + `dynamic_decode` will mix up the finished state across these entries because + it does not track the reshuffle across time steps. In this case, it is + up to the decoder to declare that it will keep track of its own finished + state by setting this property to `True`. + + Returns: + Python bool. + """ + return False + def _create_zero_outputs(size, dtype, batch_size): """Create a zero outputs Tensor structure.""" @@ -232,7 +252,10 @@ def dynamic_decode(decoder, """ (next_outputs, decoder_state, next_inputs, decoder_finished) = decoder.step(time, inputs, state) - next_finished = math_ops.logical_or(decoder_finished, finished) + if decoder.tracks_own_finished: + next_finished = decoder_finished + else: + next_finished = math_ops.logical_or(decoder_finished, finished) if maximum_iterations is not None: next_finished = math_ops.logical_or( next_finished, time + 1 >= maximum_iterations) -- GitLab From a86a589c8b329176bfbb64552405644cb641d99e Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Tue, 17 Oct 2017 09:39:50 -0700 Subject: [PATCH 057/573] Disable flaky cluster tests in opensource. PiperOrigin-RevId: 172477381 --- tensorflow/contrib/opt/BUILD | 3 +++ tensorflow/python/BUILD | 6 +++++- tensorflow/python/debug/BUILD | 2 +- tensorflow/python/kernel_tests/BUILD | 5 ++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index b5a67206f3..8b2b31d5bc 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -145,6 +145,9 @@ tf_py_test( "//tensorflow/python:training", "//tensorflow/python:variables", ], + tags = [ + "no_oss", # Flaky due to port collisions + ], ) filegroup( diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 48436fe8cf..f4106ac68c 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3197,7 +3197,10 @@ cuda_py_test( ":variables", "//third_party/py/numpy", ], - tags = ["oss_serial"], + tags = [ + "no_oss", # Test flaky due to port collisions. + "oss_serial", + ], ) tf_py_test( @@ -3213,6 +3216,7 @@ tf_py_test( ":variables", ], tags = [ + "no_oss", # Test flaky due to port collisions. "notsan", # data race due to b/62910646 "oss_serial", ], diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index b68b6e05b6..68b97ddbe3 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -897,8 +897,8 @@ cuda_py_test( "//tensorflow/python:variables", ], tags = [ + "no_oss", # Test flaky due to port collisions. "no_windows", - "nomac", # TODO(cais): Install of futures and grpcio on all macs. "notsan", "oss_serial", ], diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 1380ef5b6a..f6ecd1f0b8 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -3152,7 +3152,10 @@ tf_py_test( "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/ops:iterator_ops", ], - tags = ["no_windows"], + tags = [ + "no_oss", # Test flaky due to port collisions. + "no_windows", + ], ) filegroup( -- GitLab From f8b3ced20f7063b3c8efb0e691f28bef845a05f6 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 17 Oct 2017 09:43:46 -0700 Subject: [PATCH 058/573] Reworks the imperative_grad interface. PiperOrigin-RevId: 172477878 --- tensorflow/python/eager/backprop.py | 47 ++++------------------ tensorflow/python/eager/backprop_test.py | 15 +++---- tensorflow/python/eager/imperative_grad.py | 19 ++++++++- 3 files changed, 33 insertions(+), 48 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 1819fba4cb..61c905f31e 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -681,48 +681,17 @@ def _aggregate_grads(gradients): return ops.IndexedSlices(values, indices, dense_shape) -# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total -# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation -# so as to release the gradient tensor to save memory. -_MIN_AGGREGATE_COUNT = 4 -_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024 - - -def _add_new_grads(gradients, gradients_size, tid, grad): - """Adds a new gradient and maybe aggregate the gradients. - - Args: - gradients: A dict map from tensor id to list of gradients. - gradients_size: A dict map from tensor id to its total units. Might - not be initialized. - tid: Tensor id. - grad: New gradient for the `tid`, either a Tensor or IndexedSlices. - - Raises: - ValueError: if `grad` is neight Tensor nor IndexedSlices. - """ - tensor_grads = gradients[tid] - tensor_grads.append(grad) - if len(tensor_grads) < _MIN_AGGREGATE_COUNT: - return - elif tid not in gradients_size: - if isinstance(grad, ops.Tensor): - size = functools.reduce(operator.mul, grad._shape_tuple(), 1) # pylint: disable=protected-access - elif isinstance(grad, ops.IndexedSlices): - size = functools.reduce(operator.mul, grad.values._shape_tuple(), 1) # pylint: disable=protected-access - else: - raise ValueError("Unexpected gradient type: %s" % type(grad)) - gradients_size[tid] = size - else: - size = gradients_size[tid] - - # For simplicity, assume each element to be 4 bytes now. - if len(tensor_grads) * size * 4 > _MIN_AGGREGATE_BYTES: - gradients[tid] = [_aggregate_grads(tensor_grads)] +def _num_elements(grad): + """The number of elements in the `grad` tensor.""" + if isinstance(grad, ops.Tensor): + return functools.reduce(operator.mul, grad._shape_tuple(), 1) # pylint: disable=protected-access + if isinstance(grad, ops.IndexedSlices): + return functools.reduce(operator.mul, grad.values._shape_tuple(), 1) # pylint: disable=protected-access + raise ValueError("`grad` not a Tensor or IndexedSlices.") _default_vspace = imperative_grad.VSpace( - add_new_grads_fn=_add_new_grads, + num_elements_fn=_num_elements, aggregate_fn=_aggregate_grads, tensor_id=ops.tensor_id, zeros=array_ops.zeros, diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 9083e3a712..2645d542c0 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -22,6 +22,7 @@ from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.eager import custom_gradient +from tensorflow.python.eager import imperative_grad from tensorflow.python.eager import tape from tensorflow.python.eager import test from tensorflow.python.framework import constant_op @@ -442,21 +443,21 @@ class BackpropTest(test.TestCase): # Reduce the aggregation limit, cause the backprop to do some # early aggregation. # pylint: disable=protected-access - old_cnt = backprop._MIN_AGGREGATE_COUNT - old_bytes = backprop._MIN_AGGREGATE_BYTES - backprop._MIN_AGGREGATE_COUNT = 10 - backprop._MIN_AGGREGATE_BYTES = 1 + old_cnt = imperative_grad._MIN_AGGREGATE_COUNT + old_bytes = imperative_grad._MIN_AGGREGATE_BYTES + imperative_grad._MIN_AGGREGATE_COUNT = 10 + imperative_grad._MIN_AGGREGATE_BYTES = 1 _ = backprop.implicit_grad(fn)() self.assertEqual(len(add_n), 6) del add_n[:] # Aggregation is also limited by the memory. - backprop._MIN_AGGREGATE_BYTES = 10000 + imperative_grad._MIN_AGGREGATE_BYTES = 10000 _ = backprop.implicit_grad(fn)() self.assertEqual(len(add_n), 2) - backprop._MIN_AGGREGATE_COUNT = old_cnt - backprop._MIN_AGGREGATE_BYTES = old_bytes + imperative_grad._MIN_AGGREGATE_COUNT = old_cnt + imperative_grad._MIN_AGGREGATE_BYTES = old_bytes # pylint: enable=protected-access context.context().clear_post_execution_callbacks() diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py index d30d124040..ce58e661d7 100644 --- a/tensorflow/python/eager/imperative_grad.py +++ b/tensorflow/python/eager/imperative_grad.py @@ -120,7 +120,14 @@ def _initial_gradients(vspace, target, output_gradients, tensor_usage_counts): VSpace = collections.namedtuple( "VSpace", - ["add_new_grads_fn", "aggregate_fn", "tensor_id", "zeros", "ones_like"]) + ["aggregate_fn", "num_elements_fn", "tensor_id", "zeros", "ones_like"]) + + +# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total +# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation +# so as to release the gradient tensor to save memory. +_MIN_AGGREGATE_COUNT = 4 +_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024 def imperative_grad( @@ -193,7 +200,15 @@ def imperative_grad( in_gradients = op_trace.backward_function(*(out_gradients)) for i, t in enumerate(op_trace.input_ids): if in_gradients[i] is not None: - vspace.add_new_grads_fn(gradients, gradients_size, t, in_gradients[i]) + t_grads = gradients.setdefault(t, []) + t_grads.append(in_gradients[i]) + if len(t_grads) >= _MIN_AGGREGATE_COUNT: + if t not in gradients_size: + gradients_size[t] = vspace.num_elements_fn(t_grads[-1]) + size = gradients_size[t] + + if len(t_grads) * size * 4 > _MIN_AGGREGATE_BYTES: + t_grads[:] = [vspace.aggregate_fn(t_grads)] if tensor_usage_counts.get(t, 0) > 0: tensor_usage_counts[t] -= 1 if (t in tensor_to_op -- GitLab From 4cc3cfe15f8ed29323c669c4cfff7c754fb59136 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Tue, 17 Oct 2017 10:08:16 -0700 Subject: [PATCH 059/573] Create a macos py3 test script. PiperOrigin-RevId: 172480793 --- .../tools/ci_build/osx/cpu/run_py3_cc_core.sh | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh new file mode 100755 index 0000000000..8f839ca110 --- /dev/null +++ b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +set -e +set -x + +N_JOBS=$(sysctl -n hw.ncpu) +N_JOBS=$((N_JOBS+1)) + +echo "" +echo "Bazel will use ${N_JOBS} concurrent job(s)." +echo "" + +# Run configure. +export TF_NEED_CUDA=0 +export PYTHON_BIN_PATH=$(which python3) +yes "" | $PYTHON_BIN_PATH configure.py +which bazel +bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac \ + --test_timeout 300,450,1200,3600 \ + --test_size_filters=small,medium \ + --jobs=${N_JOBS} --build_tests_only --test_output=errors -k -- \ + //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... -- GitLab From de2a766ba6314ca82635d8bfa00bf1075cd68d64 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Tue, 17 Oct 2017 10:13:31 -0700 Subject: [PATCH 060/573] Disable flaky gru_ops_test in opensource. PiperOrigin-RevId: 172481341 --- tensorflow/contrib/rnn/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD index 6395cd8316..571d299ad9 100644 --- a/tensorflow/contrib/rnn/BUILD +++ b/tensorflow/contrib/rnn/BUILD @@ -277,6 +277,7 @@ cuda_py_tests( "//tensorflow/python:variable_scope", "//tensorflow/python:variables", ], + tags = ["no_oss"], ) tf_cc_test( -- GitLab From c4fd2362bc73b48add49d7af0ddd9ea7b6409323 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 17 Oct 2017 11:18:59 -0700 Subject: [PATCH 061/573] Fix TF workspace issue that prevents submodules from using aws build targets correctly. Fixes https://github.com/tensorflow/serving/issues/615 PiperOrigin-RevId: 172489253 --- tensorflow/workspace.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 6151dc6241..54559edbea 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -661,6 +661,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "f599b57aec4f03ad696044dd430b2d201864113937353adc346f53ad47991319", strip_prefix = "aws-sdk-cpp-1.0.90", build_file = str(Label("//third_party:aws.BUILD")), + repository = tf_repo_name ) java_import_external( -- GitLab From 476ef197b86f1ab42a73ac4ab50080953578a161 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 11:53:00 -0700 Subject: [PATCH 062/573] Automated g4 rollback of changelist 172051437 PiperOrigin-RevId: 172493077 --- tensorflow/cc/saved_model/BUILD | 1 - tensorflow/cc/saved_model/loader.cc | 16 - tensorflow/cc/saved_model/loader_test.cc | 24 +- .../00000123/assets/foo.txt | 1 - .../00000123/saved_model.pbtxt | 2728 ----------------- .../variables/variables.data-00000-of-00001 | Bin 12 -> 0 bytes .../00000123/variables/variables.index | Bin 151 -> 0 bytes 7 files changed, 2 insertions(+), 2768 deletions(-) delete mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt delete mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt delete mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 delete mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD index 67b2e4b81a..d29ad3ebcb 100644 --- a/tensorflow/cc/saved_model/BUILD +++ b/tensorflow/cc/saved_model/BUILD @@ -89,7 +89,6 @@ tf_cc_test( filegroup( name = "saved_model_half_plus_two", srcs = glob([ - "testdata/half_plus_two_forward_compatibility/**", "testdata/half_plus_two_pbtxt/**", "testdata/half_plus_two_main_op/**", "testdata/half_plus_two/**", diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc index 462308a48f..f98abc8a81 100644 --- a/tensorflow/cc/saved_model/loader.cc +++ b/tensorflow/cc/saved_model/loader.cc @@ -18,7 +18,6 @@ limitations under the License. #include #include "tensorflow/cc/saved_model/constants.h" -#include "tensorflow/core/framework/graph_def_util.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/monitoring/counter.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -225,18 +224,6 @@ Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def, return Status::OK(); } -// For forward compatibility, remove new default attributes from the graph def -// that were not present in the consumer (e.g. If graph was exported using -// code that's newer than the server and a new default attr was added). -Status RemoveNewDefaultAttrsFromMetaGraphDef(MetaGraphDef* meta_graph_def) { - OpListOpRegistry producer_op_registry( - &meta_graph_def->meta_info_def().stripped_op_list()); - OpRegistry* consumer_op_registry = OpRegistry::Global(); - return RemoveNewDefaultAttrsFromGraphDef(meta_graph_def->mutable_graph_def(), - *consumer_op_registry, - producer_op_registry, nullptr); -} - Status LoadSavedModelInternal(const SessionOptions& session_options, const RunOptions& run_options, const string& export_dir, @@ -254,9 +241,6 @@ Status LoadSavedModelInternal(const SessionOptions& session_options, TF_RETURN_IF_ERROR( FindMetaGraphDefToLoad(saved_model_proto, tags, &bundle->meta_graph_def)); - TF_RETURN_IF_ERROR( - RemoveNewDefaultAttrsFromMetaGraphDef(&bundle->meta_graph_def)); - TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession( bundle->meta_graph_def, session_options, &bundle->session)); diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc index 6dd14837b5..0ad6b33bba 100644 --- a/tensorflow/cc/saved_model/loader_test.cc +++ b/tensorflow/cc/saved_model/loader_test.cc @@ -29,12 +29,10 @@ limitations under the License. namespace tensorflow { namespace { -constexpr char kTestDataForwardCompatibility[] = - "cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123"; -constexpr char kTestDataMainOp[] = - "cc/saved_model/testdata/half_plus_two_main_op/00000123"; constexpr char kTestDataPbTxt[] = "cc/saved_model/testdata/half_plus_two_pbtxt/00000123"; +constexpr char kTestDataMainOp[] = + "cc/saved_model/testdata/half_plus_two_main_op/00000123"; constexpr char kTestDataSharded[] = "cc/saved_model/testdata/half_plus_two/00000123"; @@ -169,24 +167,6 @@ TEST_F(LoaderTest, PbtxtFormat) { CheckSavedModelBundle(export_dir, bundle); } -// Forward compatibility graph has a new attr with a default value equal to the -// value used by the server. If we handle new default attrs correctly, this test -// will pass. This simulates adding new atts to the training code while server -// code lags behind. -TEST_F(LoaderTest, ForwardCompatibility) { - SavedModelBundle bundle; - SessionOptions session_options; - RunOptions run_options; - - // TODO(b/67753689): Add support for regenerating this model in the export - // code. - const string export_dir = - io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataForwardCompatibility); - TF_ASSERT_OK(LoadSavedModel(session_options, run_options, export_dir, - {kSavedModelTagServe}, &bundle)); - CheckSavedModelBundle(export_dir, bundle); -} - TEST_F(LoaderTest, MainOpFormat) { SavedModelBundle bundle; SessionOptions session_options; diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt deleted file mode 100644 index f9ff036688..0000000000 --- a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt +++ /dev/null @@ -1 +0,0 @@ -asset-file-contents \ No newline at end of file diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt deleted file mode 100755 index e799b3579c..0000000000 --- a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt +++ /dev/null @@ -1,2728 +0,0 @@ -saved_model_schema_version: 1 -meta_graphs { - meta_info_def { - stripped_op_list { - op { - name: "Add" - input_arg { - name: "x" - type_attr: "T" - } - input_arg { - name: "y" - type_attr: "T" - } - output_arg { - name: "z" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_FLOAT - type: DT_DOUBLE - type: DT_UINT8 - type: DT_INT8 - type: DT_INT16 - type: DT_INT32 - type: DT_INT64 - type: DT_COMPLEX64 - type: DT_COMPLEX128 - type: DT_STRING - } - } - } - } - op { - name: "Assign" - input_arg { - name: "ref" - type_attr: "T" - is_ref: true - } - input_arg { - name: "value" - type_attr: "T" - } - output_arg { - name: "output_ref" - type_attr: "T" - is_ref: true - } - attr { - name: "T" - type: "type" - } - attr { - name: "validate_shape" - type: "bool" - default_value { - b: true - } - } - attr { - name: "use_locking" - type: "bool" - default_value { - b: true - } - } - allows_uninitialized_input: true - } - op { - name: "Const" - output_arg { - name: "output" - type_attr: "dtype" - } - attr { - name: "value" - type: "tensor" - } - attr { - name: "dtype" - type: "type" - } - } - op { - name: "Identity" - input_arg { - name: "input" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - } - } - op { - name: "MergeV2Checkpoints" - input_arg { - name: "checkpoint_prefixes" - type: DT_STRING - } - input_arg { - name: "destination_prefix" - type: DT_STRING - } - attr { - name: "delete_old_dirs" - type: "bool" - default_value { - b: true - } - } - } - op { - name: "Mul" - input_arg { - name: "x" - type_attr: "T" - } - input_arg { - name: "y" - type_attr: "T" - } - output_arg { - name: "z" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_FLOAT - type: DT_DOUBLE - type: DT_UINT8 - type: DT_INT8 - type: DT_UINT16 - type: DT_INT16 - type: DT_INT32 - type: DT_INT64 - type: DT_COMPLEX64 - type: DT_COMPLEX128 - } - } - } - is_commutative: true - } - op { - name: "NoOp" - } - op { - name: "Pack" - input_arg { - name: "values" - type_attr: "T" - number_attr: "N" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "N" - type: "int" - has_minimum: true - minimum: 1 - } - attr { - name: "T" - type: "type" - } - attr { - name: "axis" - type: "int" - default_value { - i: 0 - } - } - } - op { - name: "ParseExample" - input_arg { - name: "serialized" - type_attr: "TInputs" - } - input_arg { - name: "names" - type: DT_STRING - } - input_arg { - name: "sparse_keys" - type: DT_STRING - number_attr: "Nsparse" - } - input_arg { - name: "dense_keys" - type: DT_STRING - number_attr: "Ndense" - } - input_arg { - name: "dense_defaults" - type_list_attr: "Tdense" - } - output_arg { - name: "sparse_indices" - type: DT_INT64 - number_attr: "Nsparse" - } - output_arg { - name: "sparse_values" - type_list_attr: "sparse_types" - } - output_arg { - name: "sparse_shapes" - type: DT_INT64 - number_attr: "Nsparse" - } - output_arg { - name: "dense_values" - type_list_attr: "Tdense" - } - attr { - name: "Nsparse" - type: "int" - has_minimum: true - } - attr { - name: "TInputs" - type: "type" - default_value { - type: DT_STRING - } - allowed_values { - list { - type: DT_STRING - type: DT_INT64 - } - } - } - attr { - name: "Ndense" - type: "int" - has_minimum: true - } - attr { - name: "sparse_types" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "Tdense" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "dense_shapes" - type: "list(shape)" - has_minimum: true - } - } - op { - name: "Placeholder" - output_arg { - name: "output" - type_attr: "dtype" - } - attr { - name: "dtype" - type: "type" - } - attr { - name: "shape" - type: "shape" - default_value { - shape { - unknown_rank: true - } - } - } - } - op { - name: "Reshape" - input_arg { - name: "tensor" - type_attr: "T" - } - input_arg { - name: "shape" - type_attr: "Tshape" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - } - attr { - name: "Tshape" - type: "type" - default_value { - type: DT_INT32 - } - allowed_values { - list { - type: DT_INT32 - type: DT_INT64 - } - } - } - } - op { - name: "RestoreV2" - input_arg { - name: "prefix" - type: DT_STRING - } - input_arg { - name: "tensor_names" - type: DT_STRING - } - input_arg { - name: "shape_and_slices" - type: DT_STRING - } - output_arg { - name: "tensors" - type_list_attr: "dtypes" - } - attr { - name: "dtypes" - type: "list(type)" - has_minimum: true - minimum: 1 - } - } - op { - name: "SaveV2" - input_arg { - name: "prefix" - type: DT_STRING - } - input_arg { - name: "tensor_names" - type: DT_STRING - } - input_arg { - name: "shape_and_slices" - type: DT_STRING - } - input_arg { - name: "tensors" - type_list_attr: "dtypes" - } - attr { - name: "dtypes" - type: "list(type)" - has_minimum: true - minimum: 1 - } - } - op { - name: "ShardedFilename" - input_arg { - name: "basename" - type: DT_STRING - } - input_arg { - name: "shard" - type: DT_INT32 - } - input_arg { - name: "num_shards" - type: DT_INT32 - } - output_arg { - name: "filename" - type: DT_STRING - } - } - op { - name: "StringJoin" - input_arg { - name: "inputs" - type: DT_STRING - number_attr: "N" - } - output_arg { - name: "output" - type: DT_STRING - } - attr { - name: "N" - type: "int" - has_minimum: true - minimum: 1 - } - attr { - name: "separator" - type: "string" - default_value { - s: "" - } - } - } - op { - name: "VariableV2" - output_arg { - name: "ref" - type_attr: "dtype" - is_ref: true - } - attr { - name: "shape" - type: "shape" - } - attr { - name: "dtype" - type: "type" - } - attr { - name: "container" - type: "string" - default_value { - s: "" - } - } - attr { - name: "shared_name" - type: "string" - default_value { - s: "" - } - } - is_stateful: true - } - } - tags: "serve" - tensorflow_version: "1.1.0-rc2" - tensorflow_git_version: "unknown" - } - graph_def { - node { - name: "a/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - } - float_val: 0.5 - } - } - } - } - node { - name: "a" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "a/Assign" - op: "Assign" - input: "a" - input: "a/initial_value" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@a" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "a/read" - op: "Identity" - input: "a" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@a" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "b/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - } - float_val: 2.0 - } - } - } - } - node { - name: "b" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "b/Assign" - op: "Assign" - input: "b" - input: "b/initial_value" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@b" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "b/read" - op: "Identity" - input: "b" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@b" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "c/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - } - float_val: 3.0 - } - } - } - } - node { - name: "c" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "c/Assign" - op: "Assign" - input: "c" - input: "c/initial_value" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@c" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "c/read" - op: "Identity" - input: "c" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@c" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "tf_example" - op: "Placeholder" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "shape" - value { - shape { - unknown_rank: true - } - } - } - } - node { - name: "ParseExample/Const" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - dim { - } - } - } - } - } - } - node { - name: "ParseExample/key_x2" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - dim { - size: 1 - } - } - float_val: 0.0 - } - } - } - } - node { - name: "ParseExample/Reshape/shape" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - dim { - size: 1 - } - } - int_val: 1 - } - } - } - } - node { - name: "ParseExample/Reshape" - op: "Reshape" - input: "ParseExample/key_x2" - input: "ParseExample/Reshape/shape" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "Tshape" - value { - type: DT_INT32 - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - } - node { - name: "ParseExample/ParseExample/names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - } - } - } - } - } - } - node { - name: "ParseExample/ParseExample/dense_keys_0" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "x" - } - } - } - } - node { - name: "ParseExample/ParseExample/dense_keys_1" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "x2" - } - } - } - } - node { - name: "ParseExample/ParseExample" - op: "ParseExample" - input: "tf_example" - input: "ParseExample/ParseExample/names" - input: "ParseExample/ParseExample/dense_keys_0" - input: "ParseExample/ParseExample/dense_keys_1" - input: "ParseExample/Const" - input: "ParseExample/Reshape" - attr { - key: "Ndense" - value { - i: 2 - } - } - attr { - key: "TInputs" - value { - type: DT_STRING - } - } - attr { - key: "Nsparse" - value { - i: 0 - } - } - attr { - key: "Tdense" - value { - list { - type: DT_FLOAT - type: DT_FLOAT - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - attr { - key: "dense_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "sparse_types" - value { - list { - } - } - } - } - node { - name: "x" - op: "Identity" - input: "ParseExample/ParseExample" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Mul" - op: "Mul" - input: "a/read" - input: "x" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "y" - op: "Add" - input: "Mul" - input: "b/read" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Mul_1" - op: "Mul" - input: "a/read" - input: "x" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "y2" - op: "Add" - input: "Mul_1" - input: "c/read" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "x2" - op: "Identity" - input: "ParseExample/ParseExample:1" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Mul_2" - op: "Mul" - input: "a/read" - input: "x2" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "y3" - op: "Add" - input: "Mul_2" - input: "c/read" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Const" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "/tmp/original/export/assets/foo.txt" - } - } - } - } - node { - name: "filename_tensor/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "foo.txt" - } - } - } - } - node { - name: "filename_tensor" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "filename_tensor/Assign" - op: "Assign" - input: "filename_tensor" - input: "filename_tensor/initial_value" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@filename_tensor" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "filename_tensor/read" - op: "Identity" - input: "filename_tensor" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@filename_tensor" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "Assign/value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "foo.txt" - } - } - } - } - node { - name: "Assign" - op: "Assign" - input: "filename_tensor" - input: "Assign/value" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@filename_tensor" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: false - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "init" - op: "NoOp" - input: "^a/Assign" - input: "^b/Assign" - input: "^c/Assign" - } - node { - name: "group_deps" - op: "NoOp" - input: "^Assign" - } - node { - name: "save/Const" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "model" - } - } - } - } - node { - name: "save/StringJoin/inputs_1" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "_temp_80e928f1e0c844239d136d1ea966099d/part" - } - } - } - } - node { - name: "save/StringJoin" - op: "StringJoin" - input: "save/Const" - input: "save/StringJoin/inputs_1" - attr { - key: "N" - value { - i: 2 - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "separator" - value { - s: "" - } - } - } - node { - name: "save/num_shards" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - } - int_val: 1 - } - } - } - } - node { - name: "save/ShardedFilename/shard" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - } - int_val: 0 - } - } - } - } - node { - name: "save/ShardedFilename" - op: "ShardedFilename" - input: "save/StringJoin" - input: "save/ShardedFilename/shard" - input: "save/num_shards" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "save/SaveV2/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 3 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 3 - } - } - string_val: "a" - string_val: "b" - string_val: "c" - } - } - } - } - node { - name: "save/SaveV2/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 3 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 3 - } - } - string_val: "" - string_val: "" - string_val: "" - } - } - } - } - node { - name: "save/SaveV2" - op: "SaveV2" - input: "save/ShardedFilename" - input: "save/SaveV2/tensor_names" - input: "save/SaveV2/shape_and_slices" - input: "a" - input: "b" - input: "c" - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - type: DT_FLOAT - type: DT_FLOAT - } - } - } - } - node { - name: "save/control_dependency" - op: "Identity" - input: "save/ShardedFilename" - input: "^save/SaveV2" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@save/ShardedFilename" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "save/MergeV2Checkpoints/checkpoint_prefixes" - op: "Pack" - input: "save/ShardedFilename" - input: "^save/control_dependency" - attr { - key: "N" - value { - i: 1 - } - } - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "axis" - value { - i: 0 - } - } - } - node { - name: "save/MergeV2Checkpoints" - op: "MergeV2Checkpoints" - input: "save/MergeV2Checkpoints/checkpoint_prefixes" - input: "save/Const" - attr { - key: "delete_old_dirs" - value { - b: true - } - } - } - node { - name: "save/Identity" - op: "Identity" - input: "save/Const" - input: "^save/control_dependency" - input: "^save/MergeV2Checkpoints" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "save/RestoreV2/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "a" - } - } - } - } - node { - name: "save/RestoreV2/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "" - } - } - } - } - node { - name: "save/RestoreV2" - op: "RestoreV2" - input: "save/Const" - input: "save/RestoreV2/tensor_names" - input: "save/RestoreV2/shape_and_slices" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - } - } - } - } - node { - name: "save/Assign" - op: "Assign" - input: "a" - input: "save/RestoreV2" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@a" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "save/RestoreV2_1/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "b" - } - } - } - } - node { - name: "save/RestoreV2_1/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "" - } - } - } - } - node { - name: "save/RestoreV2_1" - op: "RestoreV2" - input: "save/Const" - input: "save/RestoreV2_1/tensor_names" - input: "save/RestoreV2_1/shape_and_slices" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - } - } - } - } - node { - name: "save/Assign_1" - op: "Assign" - input: "b" - input: "save/RestoreV2_1" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@b" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "save/RestoreV2_2/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "c" - } - } - } - } - node { - name: "save/RestoreV2_2/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "" - } - } - } - } - node { - name: "save/RestoreV2_2" - op: "RestoreV2" - input: "save/Const" - input: "save/RestoreV2_2/tensor_names" - input: "save/RestoreV2_2/shape_and_slices" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - } - } - } - } - node { - name: "save/Assign_2" - op: "Assign" - input: "c" - input: "save/RestoreV2_2" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@c" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "save/restore_shard" - op: "NoOp" - input: "^save/Assign" - input: "^save/Assign_1" - input: "^save/Assign_2" - } - node { - name: "save/restore_all" - op: "NoOp" - input: "^save/restore_shard" - } - versions { - producer: 23 - } - } - saver_def { - filename_tensor_name: "save/Const:0" - save_tensor_name: "save/Identity:0" - restore_op_name: "save/restore_all" - max_to_keep: 5 - sharded: true - keep_checkpoint_every_n_hours: 10000.0 - version: V2 - } - collection_def { - key: "asset_filepaths" - value { - node_list { - value: "Const:0" - } - } - } - collection_def { - key: "legacy_init_op" - value { - node_list { - value: "group_deps" - } - } - } - collection_def { - key: "saved_model_assets" - value { - any_list { - value { - type_url: "type.googleapis.com/tensorflow.AssetFileDef" - value: "\n\t\n\007Const:0\022\007foo.txt" - } - } - } - } - collection_def { - key: "trainable_variables" - value { - bytes_list { - value: "\n\003a:0\022\010a/Assign\032\010a/read:0" - value: "\n\003b:0\022\010b/Assign\032\010b/read:0" - value: "\n\003c:0\022\010c/Assign\032\010c/read:0" - } - } - } - collection_def { - key: "variables" - value { - bytes_list { - value: "\n\003a:0\022\010a/Assign\032\010a/read:0" - value: "\n\003b:0\022\010b/Assign\032\010b/read:0" - value: "\n\003c:0\022\010c/Assign\032\010c/read:0" - } - } - } - signature_def { - key: "classify_x2_to_y3" - value { - inputs { - key: "inputs" - value { - name: "x2:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - outputs { - key: "scores" - value { - name: "y3:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/classify" - } - } - signature_def { - key: "classify_x_to_y" - value { - inputs { - key: "inputs" - value { - name: "tf_example:0" - dtype: DT_STRING - tensor_shape { - unknown_rank: true - } - } - } - outputs { - key: "scores" - value { - name: "y:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/classify" - } - } - signature_def { - key: "regress_x2_to_y3" - value { - inputs { - key: "inputs" - value { - name: "x2:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - outputs { - key: "outputs" - value { - name: "y3:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/regress" - } - } - signature_def { - key: "regress_x_to_y" - value { - inputs { - key: "inputs" - value { - name: "tf_example:0" - dtype: DT_STRING - tensor_shape { - unknown_rank: true - } - } - } - outputs { - key: "outputs" - value { - name: "y:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/regress" - } - } - signature_def { - key: "regress_x_to_y2" - value { - inputs { - key: "inputs" - value { - name: "tf_example:0" - dtype: DT_STRING - tensor_shape { - unknown_rank: true - } - } - } - outputs { - key: "outputs" - value { - name: "y2:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/regress" - } - } - signature_def { - key: "serving_default" - value { - inputs { - key: "x" - value { - name: "x:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - outputs { - key: "y" - value { - name: "y:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/predict" - } - } -} diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 deleted file mode 100755 index 15b75d6ef6bffc336d138d923badb3928b8c4c13..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmZQzV6bOkU~phyaBu(s1VaG; diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index deleted file mode 100755 index 7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 151 zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5 Date: Tue, 17 Oct 2017 12:22:20 -0700 Subject: [PATCH 063/573] Fix #13731 by adding HistogramdFixedWidth in hidden_ops.txt and create the python wrapper (#13781) * Fix 13731 by adding HistogramdFixedWidth in hidden_ops.txt and create the python wrapper so that both api compatibility and test utility code in contrib could be preserved. See https://github.com/tensorflow/tensorflow/pull/13731#issuecomment-337186002 for reference. Signed-off-by: Yong Tang * Add underscore (`_histogram_fixed_width`) in calling gen_math_ops.py Signed-off-by: Yong Tang * clang-format -i histogram_op.cc Signed-off-by: Yong Tang --- tensorflow/core/kernels/histogram_op.cc | 24 ++++++++++++------------ tensorflow/core/ops/math_ops.cc | 2 +- tensorflow/python/ops/hidden_ops.txt | 1 + tensorflow/python/ops/histogram_ops.py | 4 ++-- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc index c170f172e4..4e035286f6 100644 --- a/tensorflow/core/kernels/histogram_op.cc +++ b/tensorflow/core/kernels/histogram_op.cc @@ -74,45 +74,44 @@ struct HistogramFixedWidthFunctor { template class HistogramFixedWidthOp : public OpKernel { public: - explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("nbins", &nbins_)); - OP_REQUIRES( - ctx, (nbins_ > 0), - errors::InvalidArgument("nbins should be a positive number, but got '", - nbins_, "'")); - } + explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} void Compute(OpKernelContext* ctx) override { const Tensor& values_tensor = ctx->input(0); const Tensor& value_range_tensor = ctx->input(1); + const Tensor& nbins_tensor = ctx->input(2); OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()), errors::InvalidArgument("value_range should be a vector.")); OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2), errors::InvalidArgument( "value_range should be a vector of 2 elements.")); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(nbins_tensor.shape()), + errors::InvalidArgument("nbins should be a scalar.")); const auto values = values_tensor.flat(); const auto value_range = value_range_tensor.flat(); + const auto nbins = nbins_tensor.scalar()(); OP_REQUIRES( ctx, (value_range(0) < value_range(1)), errors::InvalidArgument("value_range should satisfy value_range[0] < " "value_range[1], but got '[", value_range(0), ", ", value_range(1), "]'")); + OP_REQUIRES( + ctx, (nbins > 0), + errors::InvalidArgument("nbins should be a positive number, but got '", + nbins, "'")); Tensor* out_tensor; OP_REQUIRES_OK(ctx, - ctx->allocate_output(0, TensorShape({nbins_}), &out_tensor)); + ctx->allocate_output(0, TensorShape({nbins}), &out_tensor)); auto out = out_tensor->flat(); OP_REQUIRES_OK( ctx, functor::HistogramFixedWidthFunctor::Compute( - ctx, values, value_range, nbins_, out)); + ctx, values, value_range, nbins, out)); } - - private: - int nbins_; }; #define REGISTER_KERNELS(type) \ @@ -135,6 +134,7 @@ TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS); REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ .Device(DEVICE_GPU) \ .HostMemory("value_range") \ + .HostMemory("nbins") \ .TypeConstraint("T") \ .TypeConstraint("dtype"), \ HistogramFixedWidthOp) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index a1c608ee54..61db896c51 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -2253,8 +2253,8 @@ product: Pairwise cross product of the vectors in `a` and `b`. REGISTER_OP("HistogramFixedWidth") .Input("values: T") .Input("value_range: T") + .Input("nbins: int32") .Output("out: dtype") - .Attr("nbins: int = 100") .Attr("T: {int32, int64, float32, float64}") .Attr("dtype: {int32, int64} = DT_INT32") .SetShapeFn([](InferenceContext* c) { diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt index 04dfb5b65d..86bc038e86 100644 --- a/tensorflow/python/ops/hidden_ops.txt +++ b/tensorflow/python/ops/hidden_ops.txt @@ -259,6 +259,7 @@ ComplexAbs Conj FloorDiv FloorMod +HistogramFixedWidth Max Mean Min diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py index 040c3a5ae8..51e4be9343 100644 --- a/tensorflow/python/ops/histogram_ops.py +++ b/tensorflow/python/ops/histogram_ops.py @@ -71,5 +71,5 @@ def histogram_fixed_width(values, """ with ops.name_scope(name, 'histogram_fixed_width', [values, value_range, nbins]) as name: - return gen_math_ops.histogram_fixed_width(values, value_range, nbins, - dtype=dtype, name=name) + return gen_math_ops._histogram_fixed_width(values, value_range, nbins, + dtype=dtype, name=name) -- GitLab From 568127ac3b8e501bb230ee287ec9a46129fad349 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 17 Oct 2017 12:23:29 -0700 Subject: [PATCH 064/573] Improve shape inference with `DecodeAndCropJpeg` (#13750) * Improve shape inference with `DecodeAndCropJpeg` While working on improving shape inference for several other ops in 13561 and 13193, I noticed that `DecodeAndCropJpeg` does not inference shape even though crop size might have already be provided. In that case the shape will be `[h, w, channel]` and `h`, `w` is part of the `crop_window`. This fix updates the shape function in `DecodeAndCropJpeg` for improving shape inference. Signed-off-by: Yong Tang * Add test cases to cover shape inference for `DecodeAndCropJpeg` Signed-off-by: Yong Tang * Address failed unit tests Signed-off-by: Yong Tang --- tensorflow/core/ops/image_ops.cc | 31 ++++++++++++++++++++++++- tensorflow/core/ops/image_ops_test.cc | 6 ++--- tensorflow/python/ops/image_ops_test.py | 6 ++++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc index 66765a3333..89c9da81c5 100644 --- a/tensorflow/core/ops/image_ops.cc +++ b/tensorflow/core/ops/image_ops.cc @@ -453,7 +453,36 @@ REGISTER_OP("DecodeAndCropJpeg") .Attr("acceptable_fraction: float = 1.0") .Attr("dct_method: string = ''") .Output("image: uint8") - .SetShapeFn(DecodeImageShapeFn) + .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + DimensionHandle channels_dim = c->UnknownDim(); + DimensionHandle h = c->UnknownDim(); + DimensionHandle w = c->UnknownDim(); + + int32 channels; + TF_RETURN_IF_ERROR(c->GetAttr("channels", &channels)); + if (channels != 0) { + if (channels < 0) { + return errors::InvalidArgument("channels must be non-negative, got ", + channels); + } + channels_dim = c->MakeDim(channels); + } + + DimensionHandle unused_dim; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused)); + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(unused, 0), 4, &unused_dim)); + + const Tensor* crop_window = c->input_tensor(1); + if (crop_window != nullptr) { + auto crop_window_vec = crop_window->vec(); + h = c->MakeDim(crop_window_vec(2)); + w = c->MakeDim(crop_window_vec(3)); + } + c->set_output(0, c->MakeShape({h, w, channels_dim})); + return Status::OK(); + }) .Doc(strings::StrCat(R"doc( Decode and Crop a JPEG-encoded image to a uint8 tensor. )doc", diff --git a/tensorflow/core/ops/image_ops_test.cc b/tensorflow/core/ops/image_ops_test.cc index c34b11a15e..5f0b391b0d 100644 --- a/tensorflow/core/ops/image_ops_test.cc +++ b/tensorflow/core/ops/image_ops_test.cc @@ -105,7 +105,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) { .Input({"img", 0, DT_STRING}) .Input({"crop_window", 1, DT_INT32}) .Finalize(&op.node_def)); - INFER_OK(op, "[];[]", "[?,?,?]"); + INFER_OK(op, "[];[?]", "[?,?,?]"); // Set the channel, so that part of output shape is known. TF_ASSERT_OK(NodeDefBuilder("test", op_name) @@ -113,7 +113,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) { .Input({"crop_window", 1, DT_INT32}) .Attr("channels", 4) .Finalize(&op.node_def)); - INFER_OK(op, "[];[]", "[?,?,4]"); + INFER_OK(op, "[];[?]", "[?,?,4]"); // Negative channel value is rejected. TF_ASSERT_OK(NodeDefBuilder("test", op_name) @@ -139,7 +139,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_InvalidCropWindow) { .Input({"img", 0, DT_STRING}) .Input({"crop_window", 1, DT_INT32}) .Finalize(&op.node_def)); - INFER_OK(op, "[];[]", "[?,?,?]"); + INFER_OK(op, "[];[?]", "[?,?,?]"); } TEST(ImageOpsTest, EncodeImage_ShapeFn) { diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index 348c005ff3..b13b73edbb 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -2434,9 +2434,13 @@ class JpegTest(test_util.TensorFlowTestCase): y, x, h, w = crop_window image1_crop = image_ops.crop_to_bounding_box(image1, y, x, h, w) - # Combined crop+decode. + # Combined decode+crop. image2 = image_ops.decode_and_crop_jpeg(jpeg0, crop_window) + # Combined decode+crop should have the same shape inference + self.assertAllEqual(image1_crop.get_shape().as_list(), + image2.get_shape().as_list()) + # CropAndDecode should be equal to DecodeJpeg+Crop. image1_crop, image2 = sess.run([image1_crop, image2]) self.assertAllEqual(image1_crop, image2) -- GitLab From 7f778806153598093af01081650fe36a16a2ff56 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Tue, 17 Oct 2017 13:47:23 -0700 Subject: [PATCH 065/573] tfdbg: add missing -f flag of the pt command to the command table PiperOrigin-RevId: 172508350 --- tensorflow/docs_src/programmers_guide/debugger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md index 58154d19e7..36a016e880 100644 --- a/tensorflow/docs_src/programmers_guide/debugger.md +++ b/tensorflow/docs_src/programmers_guide/debugger.md @@ -141,6 +141,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at | **`lt`** | | **List dumped tensors.** | `lt` | | | `-n ` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` | | | `-t ` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` | +| | `-f ` | List only the tensors that pass a registered tensor filter. | `lt -f has_inf_or_nan` | | | `-s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | | | `-r` | Sort in reverse order. | `lt -r -s dump_size` | | **`pt`** | | **Print value of a dumped tensor.** | | -- GitLab From cfb13fa789bcf1cdbbf0fd38cf7568b7098ab99b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 13:58:00 -0700 Subject: [PATCH 066/573] Added an additional check on the length of the values and boundaries lists. PiperOrigin-RevId: 172510229 --- tensorflow/python/training/learning_rate_decay.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py index bb9e26d8b4..e4a7964aaf 100644 --- a/tensorflow/python/training/learning_rate_decay.py +++ b/tensorflow/python/training/learning_rate_decay.py @@ -130,8 +130,12 @@ def piecewise_constant(x, boundaries, values, name=None): Raises: ValueError: if types of `x` and `boundaries` do not match, or types of all - `values` do not match. + `values` do not match or + the number of elements in the lists does not match. """ + if len(boundaries) != len(values) - 1: + raise ValueError( + "The length of boundaries should be 1 less than the length of values") with ops.name_scope(name, "PiecewiseConstant", [x, boundaries, values, name]) as name: x = ops.convert_to_tensor(x) @@ -158,7 +162,6 @@ def piecewise_constant(x, boundaries, values, name=None): raise ValueError( "Values must have elements all with the same dtype (%s vs %s)." % ( values[0].dtype.base_dtype, v.dtype.base_dtype)) - pred_fn_pairs = {} pred_fn_pairs[x <= boundaries[0]] = lambda: values[0] pred_fn_pairs[x > boundaries[-1]] = lambda: values[-1] -- GitLab From a760b9945511e64172bc4073d3c0efa0888dc2a1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 14:04:50 -0700 Subject: [PATCH 067/573] Internal change. PiperOrigin-RevId: 172511553 --- tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index 76e8ccc62a..7491b1b2d2 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -105,6 +105,7 @@ py_test( tags = [ "no_pip_gpu", # b/63391119 "nomsan", # Takes too long to run. + "notsan", # b/67865658 ], deps = [ ":ar_model", -- GitLab From bb7869aad43e329e9fafa027a23ee96d4ccca124 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 14:11:25 -0700 Subject: [PATCH 068/573] Remove remaining nomac tags for TFBT as the memory corruption issue is fixed. PiperOrigin-RevId: 172512636 --- tensorflow/contrib/boosted_trees/BUILD | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD index 1b85c260c0..66a04d42e9 100644 --- a/tensorflow/contrib/boosted_trees/BUILD +++ b/tensorflow/contrib/boosted_trees/BUILD @@ -81,9 +81,6 @@ py_test( size = "small", srcs = ["python/utils/losses_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":losses", "//tensorflow/python:array_ops", @@ -135,7 +132,6 @@ py_test( srcs = ["python/training/functions/gbdt_batch_test.py"], srcs_version = "PY2AND3", tags = [ - "nomac", # b/63258195 "notsan", # b/62863147 ], deps = [ @@ -164,9 +160,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/model_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":model_ops_py", ":prediction_ops_py", @@ -187,9 +180,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/prediction_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":model_ops_py", ":prediction_ops_py", @@ -207,9 +197,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/quantile_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":quantile_ops_py", "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py", @@ -247,9 +234,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/stats_accumulator_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":stats_accumulator_ops_py", "//tensorflow/python:framework_ops", @@ -264,9 +248,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/training_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":model_ops_py", ":training_ops_py", -- GitLab From d69948fa09a403106a4b4c68ed00f4a8f0c10ef1 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Tue, 17 Oct 2017 14:25:33 -0700 Subject: [PATCH 069/573] Make tensor_util.constant_value_as_shape compatible with Eager mode PiperOrigin-RevId: 172514789 --- tensorflow/python/BUILD | 1 + tensorflow/python/framework/tensor_util.py | 5 +++++ tensorflow/python/framework/tensor_util_test.py | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index f4106ac68c..e4e284dcdf 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1141,6 +1141,7 @@ py_test( ":client_testlib", ":framework", ":framework_for_generated_wrappers", + ":framework_test_lib", ":math_ops", ":state_ops_gen", "//third_party/py/numpy", diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py index 63324e5977..53eba8b747 100644 --- a/tensorflow/python/framework/tensor_util.py +++ b/tensorflow/python/framework/tensor_util.py @@ -23,6 +23,7 @@ import six from tensorflow.core.framework import tensor_pb2 from tensorflow.core.framework import tensor_shape_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.util import compat @@ -769,6 +770,10 @@ def constant_value_as_shape(tensor): # pylint: disable=invalid-name Returns: A `TensorShape` based on the constant value of the given `tensor`. """ + if context.in_eager_mode(): + return tensor_shape.as_shape( + [dim if dim != -1 else None for dim in tensor.numpy()]) + shape = tensor.get_shape().with_rank(1) if tensor.get_shape() == [0]: return tensor_shape.scalar() diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py index dda72fc0c8..b4f28cfce0 100644 --- a/tensorflow/python/framework/tensor_util_test.py +++ b/tensorflow/python/framework/tensor_util_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_state_ops from tensorflow.python.ops import math_ops @@ -901,6 +902,7 @@ class ConstantValueTest(test.TestCase): class ConstantValueAsShapeTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testConstant(self): np_val = np.random.rand(3).astype(np.int32) tf_val = constant_op.constant(np_val) @@ -913,11 +915,18 @@ class ConstantValueAsShapeTest(test.TestCase): tensor_shape.TensorShape([]), tensor_util.constant_value_as_shape(tf_val)) + @test_util.run_in_graph_and_eager_modes() def testShape(self): tf_val = array_ops.shape(constant_op.constant(0.0, shape=[1, 2, 3])) c_val = tensor_util.constant_value_as_shape(tf_val) self.assertEqual(tensor_shape.TensorShape([1, 2, 3]), c_val) + @test_util.run_in_graph_and_eager_modes() + def testMinusOneBecomesNone(self): + tf_val = constant_op.constant([-1, 1, -1], shape=[3]) + c_val = tensor_util.constant_value_as_shape(tf_val) + self.assertEqual([None, 1, None], c_val.as_list()) + def testPack(self): tf_val = array_ops.stack( [constant_op.constant(16), 37, array_ops.placeholder(dtypes.int32)]) -- GitLab From cbb705f10149a11b8d17182343ef12ab2dbfd7a8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 17 Oct 2017 14:35:51 -0700 Subject: [PATCH 070/573] Fix crash when `tf.pad` is used with int64 paddings. (#13517) * Add int64 bounds support for `tf.image.pad_to_bounding_box` This fix tries to fix the issue raised in 13506 where int64 data types for bounds in `tf.image.pad_to_bounding_box` crashes. The reason of the crash is caused by the fact that int64 was directly converted into int32 without passing through kernel registeration. This fix fixes the issue by adding `typename Tpadding` to the template. --- tensorflow/core/kernels/pad_op.cc | 144 ++++++++++++++---- tensorflow/core/kernels/pad_op.h | 10 +- tensorflow/core/kernels/pad_op_gpu.cu.cc | 20 ++- tensorflow/python/kernel_tests/pad_op_test.py | 9 ++ tensorflow/python/ops/image_ops_test.py | 19 +++ 5 files changed, 158 insertions(+), 44 deletions(-) diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc index 6196c5ed93..eff3e4d92c 100644 --- a/tensorflow/core/kernels/pad_op.cc +++ b/tensorflow/core/kernels/pad_op.cc @@ -40,9 +40,9 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL -template +template class PadOp : public OpKernel { public: explicit PadOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -82,10 +82,11 @@ class PadOp : public OpKernel { // Compute the shape of the output tensor, and allocate it. TensorShape output_shape; - TTypes::ConstMatrix paddings = in1.matrix(); + typename TTypes::ConstMatrix paddings = in1.matrix(); for (int d = 0; d < fixed_dims; ++d) { - const int32 before_d = paddings(d, 0); // Pad before existing elements. - const int32 after_d = paddings(d, 1); // Pad after existing elements. + const Tpadding before_d = + paddings(d, 0); // Pad before existing elements. + const Tpadding after_d = paddings(d, 1); // Pad after existing elements. OP_REQUIRES(context, before_d >= 0 && after_d >= 0, errors::InvalidArgument("Paddings must be non-negative: ", before_d, " ", after_d)); @@ -142,32 +143,47 @@ class PadOp : public OpKernel { template void Operate(OpKernelContext* context, typename TTypes::ConstTensor input, - TTypes::ConstMatrix paddings, T pad_value, + typename TTypes::ConstMatrix paddings, T pad_value, Tensor* output) { CHECK_EQ(Dims, paddings.dimension(0)); CHECK_EQ(2, paddings.dimension(1)); - Eigen::array, Dims> paddings_array; + Eigen::array, Dims> paddings_array; for (int i = 0; i < Dims; ++i) { paddings_array[i] = {paddings(i, 0), paddings(i, 1)}; } - functor::Pad functor; + functor::Pad functor; functor(context->eigen_device(), output->tensor(), input, paddings_array, pad_value); } }; -#define REGISTER_KERNEL(type) \ - REGISTER_KERNEL_BUILDER(Name("Pad") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .HostMemory("paddings"), \ - PadOp); \ - REGISTER_KERNEL_BUILDER(Name("PadV2") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .HostMemory("paddings") \ - .HostMemory("constant_values"), \ - PadOp); +#define REGISTER_KERNEL(type) \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp); TF_CALL_POD_TYPES(REGISTER_KERNEL); #undef REGISTER_KERNEL @@ -177,11 +193,17 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL); namespace functor { #define DECLARE_GPU_SPEC(T, Dims) \ template <> \ - void Pad::operator()( \ + void Pad::operator()( \ const GPUDevice& d, typename TTypes::Tensor output, \ typename TTypes::ConstTensor input, \ Eigen::array, Dims> paddings, T pad_value); \ - extern template struct Pad; + extern template struct Pad; \ + template <> \ + void Pad::operator()( \ + const GPUDevice& d, typename TTypes::Tensor output, \ + typename TTypes::ConstTensor input, \ + Eigen::array, Dims> paddings, T pad_value); \ + extern template struct Pad; #define DECLARE_GPU_SPECS(T) \ DECLARE_GPU_SPEC(T, 0); \ @@ -202,14 +224,27 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - PadOp); \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ REGISTER_KERNEL_BUILDER(Name("PadV2") \ .Device(DEVICE_GPU) \ .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings") \ .HostMemory("constant_values"), \ - PadOp) + PadOp) \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); @@ -223,7 +258,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad") .HostMemory("input") .HostMemory("paddings") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("Pad") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("output"), + PadOp); REGISTER_KERNEL_BUILDER(Name("PadV2") .Device(DEVICE_GPU) .TypeConstraint("T") @@ -232,7 +275,16 @@ REGISTER_KERNEL_BUILDER(Name("PadV2") .HostMemory("paddings") .HostMemory("constant_values") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("PadV2") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("constant_values") + .HostMemory("output"), + PadOp); #endif #ifdef TENSORFLOW_USE_SYCL @@ -243,14 +295,27 @@ REGISTER_KERNEL_BUILDER(Name("PadV2") .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - PadOp); \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ REGISTER_KERNEL_BUILDER(Name("PadV2") \ .Device(DEVICE_SYCL) \ .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings") \ .HostMemory("constant_values"), \ - PadOp) + PadOp) \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp) TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNEL); REGISTER_KERNEL_BUILDER(Name("Pad") @@ -260,7 +325,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad") .HostMemory("input") .HostMemory("paddings") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("Pad") + .Device(DEVICE_SYCL) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("output"), + PadOp); REGISTER_KERNEL_BUILDER(Name("PadV2") .Device(DEVICE_SYCL) .TypeConstraint("T") @@ -269,8 +342,17 @@ REGISTER_KERNEL_BUILDER(Name("PadV2") .HostMemory("paddings") .HostMemory("constant_values") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("PadV2") + .Device(DEVICE_SYCL) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("constant_values") + .HostMemory("output"), + PadOp); #undef REGISTER_SYCL_KERNEL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // end namespace tensorflow diff --git a/tensorflow/core/kernels/pad_op.h b/tensorflow/core/kernels/pad_op.h index 95a7c9a3ae..ee9e0f0330 100644 --- a/tensorflow/core/kernels/pad_op.h +++ b/tensorflow/core/kernels/pad_op.h @@ -25,13 +25,13 @@ namespace tensorflow { namespace functor { // Functor used by PadOp to do the computations. -template +template struct Pad { // Pad "input" into "output", as specified by "paddings" and "pad_value". // See pad_op.cc for details. void operator()(const Device& d, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - Eigen::array, Dims> paddings, + Eigen::array, Dims> paddings, T pad_value) { if (Eigen::internal::is_same::value && (output.size() <= std::numeric_limits::max())) { @@ -42,12 +42,12 @@ struct Pad { } }; -template -struct Pad { +template +struct Pad { // In the scalar case we simply copy the input. void operator()(const Device& d, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - Eigen::array, 0>, T) { + Eigen::array, 0>, T) { output.device(d) = input; } }; diff --git a/tensorflow/core/kernels/pad_op_gpu.cu.cc b/tensorflow/core/kernels/pad_op_gpu.cu.cc index f98631df17..613ad62825 100644 --- a/tensorflow/core/kernels/pad_op_gpu.cu.cc +++ b/tensorflow/core/kernels/pad_op_gpu.cu.cc @@ -26,14 +26,18 @@ namespace tensorflow { typedef Eigen::GpuDevice GPUDevice; // Definition of the GPU implementations declared in pad_op.cc. -#define DEFINE_GPU_SPECS(T) \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; +#define DEFINE_GPU_PAD_SPECS(T, Tpadding) \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; + +#define DEFINE_GPU_SPECS(T) \ + DEFINE_GPU_PAD_SPECS(T, int32) \ + DEFINE_GPU_PAD_SPECS(T, int64) TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS); diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py index ca1f3f878f..1af43e6067 100644 --- a/tensorflow/python/kernel_tests/pad_op_test.py +++ b/tensorflow/python/kernel_tests/pad_op_test.py @@ -284,6 +284,15 @@ class PadOpTest(test.TestCase): self.assertAllEqual(inp, out) self.assertShapeEqual(inp, tf_val) + def testPadTypes(self): + for dtype in [dtypes.int32, dtypes.int64]: + paddings = np.zeros((0, 2)) + inp = np.asarray(7) + with self.test_session(use_gpu=True): + tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype)) + out = tf_val.eval() + self.assertAllEqual(inp, out) + self.assertShapeEqual(inp, tf_val) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index b13b73edbb..d1554b399f 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -1374,6 +1374,25 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): y = image_ops.pad_to_bounding_box(image, 0, 0, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) + def testInt64(self): + x = [1, 2, 3, + 4, 5, 6, + 7, 8, 9] + x_shape = [3, 3, 1] + + y = [0, 0, 0, + 1, 2, 3, + 4, 5, 6, + 7, 8, 9] + y_shape = [4, 3, 1] + x = np.array(x).reshape(x_shape) + y = np.array(y).reshape(y_shape) + + i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64) + y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3]) + with self.test_session(use_gpu=True): + self.assertAllClose(y, y_tf.eval()) + def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) -- GitLab From b190a9ebfe102de586313565ecb75f03972d172f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 14:43:08 -0700 Subject: [PATCH 071/573] Preserve metadata_ when using HloInstruction::CloneWithNewOperands. PiperOrigin-RevId: 172517300 --- .../compiler/xla/service/hlo_instruction.cc | 146 +++++++++++------- .../xla/service/hlo_instruction_test.cc | 3 + 2 files changed, 93 insertions(+), 56 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 021e5881c8..f24953051a 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -962,6 +962,8 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( VLOG(3) << " " << new_operand->name(); } + std::unique_ptr clone; + // Explicitly call the factory for the instruction type. This is more robust // in the face of code changes than copying fields explicitly. This also // properly sets the user fields of the operands. @@ -984,7 +986,8 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kSort: case HloOpcode::kTanh: CHECK_EQ(new_operands.size(), 1); - return CreateUnary(shape, opcode_, new_operands[0]); + clone = CreateUnary(shape, opcode_, new_operands[0]); + break; // Binary ops. case HloOpcode::kAdd: case HloOpcode::kDivide: @@ -1007,118 +1010,148 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: CHECK_EQ(new_operands.size(), 2); - return CreateBinary(shape, opcode_, new_operands[0], new_operands[1]); + clone = CreateBinary(shape, opcode_, new_operands[0], new_operands[1]); + break; // Ternary ops. case HloOpcode::kClamp: case HloOpcode::kSelect: CHECK_EQ(new_operands.size(), 3); - return CreateTernary(shape, opcode_, new_operands[0], new_operands[1], - new_operands[2]); + clone = CreateTernary(shape, opcode_, new_operands[0], new_operands[1], + new_operands[2]); + break; // Other supported ops. case HloOpcode::kBroadcast: CHECK_EQ(new_operands.size(), 1); - return CreateBroadcast(shape, new_operands[0], dimensions_); + clone = CreateBroadcast(shape, new_operands[0], dimensions_); + break; case HloOpcode::kCall: - return CreateCall(shape, new_operands, to_apply()); + clone = CreateCall(shape, new_operands, to_apply()); + break; case HloOpcode::kCustomCall: - return CreateCustomCall(shape, new_operands, custom_call_target_); + clone = CreateCustomCall(shape, new_operands, custom_call_target_); + break; case HloOpcode::kConcatenate: - return CreateConcatenate(shape, new_operands, dimensions(0)); + clone = CreateConcatenate(shape, new_operands, dimensions(0)); + break; case HloOpcode::kConvert: CHECK_EQ(new_operands.size(), 1); - return CreateConvert(shape, new_operands[0]); + clone = CreateConvert(shape, new_operands[0]); + break; case HloOpcode::kReducePrecision: CHECK_EQ(new_operands.size(), 1); - return CreateReducePrecision(shape, new_operands[0], exponent_bits_, - mantissa_bits_); + clone = CreateReducePrecision(shape, new_operands[0], exponent_bits_, + mantissa_bits_); + break; case HloOpcode::kConvolution: CHECK_EQ(new_operands.size(), 2); - return CreateConvolve(shape, new_operands[0], new_operands[1], *window_, - *convolution_dimension_numbers_); + clone = CreateConvolve(shape, new_operands[0], new_operands[1], *window_, + *convolution_dimension_numbers_); + break; case HloOpcode::kCrossReplicaSum: CHECK_EQ(new_operands.size(), 1); - return CreateCrossReplicaSum(shape, new_operands[0]); + clone = CreateCrossReplicaSum(shape, new_operands[0]); + break; case HloOpcode::kGetTupleElement: CHECK_EQ(new_operands.size(), 1); - return CreateGetTupleElement(shape, new_operands[0], tuple_index()); + clone = CreateGetTupleElement(shape, new_operands[0], tuple_index()); + break; case HloOpcode::kMap: - return CreateMap(shape, new_operands, to_apply()); + clone = CreateMap(shape, new_operands, to_apply()); + break; case HloOpcode::kPad: CHECK_EQ(new_operands.size(), 2); - return CreatePad(shape, new_operands[0], new_operands[1], - *padding_config_); + clone = + CreatePad(shape, new_operands[0], new_operands[1], *padding_config_); + break; case HloOpcode::kReduce: CHECK_EQ(new_operands.size(), 2); - return CreateReduce(shape, new_operands[0], new_operands[1], dimensions_, - to_apply()); + clone = CreateReduce(shape, new_operands[0], new_operands[1], dimensions_, + to_apply()); + break; case HloOpcode::kReduceWindow: CHECK_EQ(new_operands.size(), 2); - return CreateReduceWindow(shape, new_operands[0], new_operands[1], - *window_, to_apply()); + clone = CreateReduceWindow(shape, new_operands[0], new_operands[1], + *window_, to_apply()); + break; case HloOpcode::kSelectAndScatter: CHECK_EQ(new_operands.size(), 3); - return CreateSelectAndScatter(shape, new_operands[0], select(), *window_, - new_operands[1], new_operands[2], - scatter()); + clone = + CreateSelectAndScatter(shape, new_operands[0], select(), *window_, + new_operands[1], new_operands[2], scatter()); + break; case HloOpcode::kReverse: CHECK_EQ(new_operands.size(), 1); - return CreateReverse(shape, new_operands[0], dimensions_); + clone = CreateReverse(shape, new_operands[0], dimensions_); + break; case HloOpcode::kRng: - return CreateRng(shape, distribution_, new_operands); + clone = CreateRng(shape, distribution_, new_operands); + break; case HloOpcode::kReshape: CHECK_EQ(new_operands.size(), 1); - return CreateReshape(shape, new_operands[0]); + clone = CreateReshape(shape, new_operands[0]); + break; case HloOpcode::kSlice: CHECK_EQ(new_operands.size(), 1); - return CreateSlice(shape, new_operands[0], slice_starts_, slice_limits_, - slice_strides_); + clone = CreateSlice(shape, new_operands[0], slice_starts_, slice_limits_, + slice_strides_); + break; case HloOpcode::kDynamicSlice: - return CreateDynamicSlice(shape, new_operands[0], new_operands[1], - dynamic_slice_sizes_); + clone = CreateDynamicSlice(shape, new_operands[0], new_operands[1], + dynamic_slice_sizes_); + break; case HloOpcode::kDynamicUpdateSlice: CHECK_EQ(new_operands.size(), 3); - return CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1], - new_operands[2]); + clone = CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1], + new_operands[2]); + break; case HloOpcode::kTranspose: CHECK_EQ(new_operands.size(), 1); - return CreateTranspose(shape, new_operands[0], dimensions_); - case HloOpcode::kTuple: { - auto new_tuple = CreateTuple(new_operands); - *new_tuple->mutable_shape() = shape; - return new_tuple; - } + clone = CreateTranspose(shape, new_operands[0], dimensions_); + break; + case HloOpcode::kTuple: + clone = CreateTuple(new_operands); + *clone->mutable_shape() = shape; + break; case HloOpcode::kWhile: CHECK_EQ(new_operands.size(), 1); - return CreateWhile(shape, while_condition(), while_body(), - new_operands[0]); + clone = + CreateWhile(shape, while_condition(), while_body(), new_operands[0]); + break; case HloOpcode::kConstant: - return CreateConstant(literal_->CloneToUnique()); + clone = CreateConstant(literal_->CloneToUnique()); + break; case HloOpcode::kFusion: - return CloneFusionWithNewOperands(shape, new_operands); + clone = CloneFusionWithNewOperands(shape, new_operands); + break; case HloOpcode::kParameter: - return CreateParameter(parameter_number_, shape, parameter_name_); + clone = CreateParameter(parameter_number_, shape, parameter_name_); + break; case HloOpcode::kBatchNormTraining: CHECK_EQ(new_operands.size(), 3); - return CreateBatchNormTraining(shape, new_operands[0], new_operands[1], - new_operands[2], epsilon(), - feature_index()); - + clone = + CreateBatchNormTraining(shape, new_operands[0], new_operands[1], + new_operands[2], epsilon(), feature_index()); + break; case HloOpcode::kBatchNormInference: CHECK_EQ(new_operands.size(), 5); - return CreateBatchNormInference( + clone = CreateBatchNormInference( shape, new_operands[0], new_operands[1], new_operands[2], new_operands[3], new_operands[4], epsilon(), feature_index()); + break; case HloOpcode::kInfeed: CHECK_EQ(new_operands.size(), 0); - return CreateInfeed(shape, infeed_config()); + clone = CreateInfeed(shape, infeed_config()); + break; case HloOpcode::kOutfeed: CHECK_EQ(new_operands.size(), 1); - return CreateOutfeed(outfeed_shape_, new_operands[0], outfeed_config()); + clone = CreateOutfeed(outfeed_shape_, new_operands[0], outfeed_config()); + break; case HloOpcode::kBatchNormGrad: CHECK_EQ(new_operands.size(), 5); - return CreateBatchNormGrad(shape, new_operands[0], new_operands[1], - new_operands[2], new_operands[3], - new_operands[4], epsilon(), feature_index()); + clone = CreateBatchNormGrad(shape, new_operands[0], new_operands[1], + new_operands[2], new_operands[3], + new_operands[4], epsilon(), feature_index()); + break; case HloOpcode::kRecv: case HloOpcode::kSend: case HloOpcode::kUpdate: @@ -1126,6 +1159,8 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kTrace: LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_); } + clone->set_metadata(metadata_); + return clone; } HloInstruction::~HloInstruction() {} @@ -1168,7 +1203,6 @@ std::unique_ptr HloInstruction::Clone( } } clone->set_parent(parent_); - clone->set_metadata(metadata_); return clone; } diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index 45f9128eab..cdafc05d8c 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -706,6 +706,9 @@ TEST_F(HloInstructionTest, PreserveMetadataInFusionAndClone) { metadata, fusion->fused_expression_root()->metadata())); EXPECT_TRUE(protobuf_util::ProtobufEquals( metadata, fusion->fused_expression_root()->operand(0)->metadata())); + + auto cloned = fusion->CloneWithNewOperands(fusion->shape(), {}); + EXPECT_TRUE(protobuf_util::ProtobufEquals(metadata, fusion->metadata())); } TEST_F(HloInstructionTest, PreserveOutfeedShapeThroughClone) { -- GitLab From e6623e3167867037fb486102deebea22e04318bd Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Tue, 17 Oct 2017 14:44:47 -0700 Subject: [PATCH 072/573] Link to Datasets doc in release notes PiperOrigin-RevId: 172517507 --- RELEASE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index c5f1e8b309..2c6535c15d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,7 +1,8 @@ # Release 1.4.0 ## Major Features And Improvements -* `tf.data` is now part of the core TensorFlow API. +* [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of + the core TensorFlow API. * The API is now subject to backwards compatibility guarantees. * For a guide to migrating from the `tf.contrib.data` API, see the [README] (https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md). -- GitLab From effa8692aebd738c50d2270f5a371528f0eca748 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 14:51:07 -0700 Subject: [PATCH 073/573] A new flag `ignore_live_threads` is available on train. If set to True, it will ignore threads that remain running when tearing down infrastructure after successfully completing training, instead of throwing a RuntimeError. PiperOrigin-RevId: 172518466 --- tensorflow/contrib/slim/python/slim/learning.py | 11 +++++++++-- tensorflow/python/training/supervisor.py | 14 +++++++++++--- .../api/golden/tensorflow.train.-supervisor.pbtxt | 4 ++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/slim/python/slim/learning.py b/tensorflow/contrib/slim/python/slim/learning.py index 5ee014a1f1..def00b7618 100644 --- a/tensorflow/contrib/slim/python/slim/learning.py +++ b/tensorflow/contrib/slim/python/slim/learning.py @@ -552,7 +552,8 @@ def train(train_op, sync_optimizer=None, session_config=None, session_wrapper=None, - trace_every_n_steps=None): + trace_every_n_steps=None, + ignore_live_threads=False): """Runs a training loop using a TensorFlow supervisor. When the sync_optimizer is supplied, gradient updates are applied @@ -615,6 +616,9 @@ def train(train_op, trace_every_n_steps: produce and save a `Timeline` in Chrome trace format and add it to the summaries every `trace_every_n_steps`. If None, no trace information will be produced or saved. + ignore_live_threads: If `True` ignores threads that remain running after + a grace period when stopping the supervisor, instead of raising a + RuntimeError. Returns: the value of the loss function after training. @@ -772,7 +776,10 @@ def train(train_op, if logdir and sv.is_chief: logging.info('Finished training! Saving model to disk.') sv.saver.save(sess, sv.save_path, global_step=sv.global_step) - sv.stop(threads, close_summary_writer=True) + sv.stop( + threads, + close_summary_writer=True, + ignore_live_threads=ignore_live_threads) except errors.AbortedError: # Always re-run on AbortedError as it indicates a restart of one of the diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index cfdd03dc15..41dbf6b497 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -768,7 +768,10 @@ class Supervisor(object): looper.start() return looper - def stop(self, threads=None, close_summary_writer=True): + def stop(self, + threads=None, + close_summary_writer=True, + ignore_live_threads=False): """Stop the services and the coordinator. This does not close the session. @@ -782,14 +785,19 @@ class Supervisor(object): close_summary_writer: Whether to close the `summary_writer`. Defaults to `True` if the summary writer was created by the supervisor, `False` otherwise. + ignore_live_threads: If `True` ignores threads that remain running after + a grace period when joining threads via the coordinator, instead of + raising a RuntimeError. """ self._coord.request_stop() try: # coord.join() re-raises the first reported exception; the "finally" # block ensures that we clean up whether or not an exception was # reported. - self._coord.join(threads, - stop_grace_period_secs=self._stop_grace_secs) + self._coord.join( + threads, + stop_grace_period_secs=self._stop_grace_secs, + ignore_live_threads=ignore_live_threads) finally: # Close the writer last, in case one of the running threads was using it. if close_summary_writer and self._summary_writer: diff --git a/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt index cc9bd5c136..1f0e59a1ac 100644 --- a/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt @@ -88,7 +88,7 @@ tf_class { } member_method { name: "Stop" - argspec: "args=[\'self\', \'threads\', \'close_summary_writer\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], " + argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], " } member_method { name: "StopOnException" @@ -136,7 +136,7 @@ tf_class { } member_method { name: "stop" - argspec: "args=[\'self\', \'threads\', \'close_summary_writer\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], " + argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], " } member_method { name: "stop_on_exception" -- GitLab From 2fb1f1d837d8f86f3ad753ea235a1b3a22ba195f Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Tue, 17 Oct 2017 14:57:04 -0700 Subject: [PATCH 074/573] Set estimator run_config default random seed to None. This will make it aligned with other parts of the TF. Many users are not aware of impact of non-random seed. For example it may lead to train only on a small fraction of training data due to preemptions. We're changing default behavior since we consider it as a bug fix. PiperOrigin-RevId: 172519268 --- tensorflow/python/estimator/run_config.py | 2 +- tensorflow/python/estimator/run_config_test.py | 2 +- .../tools/api/golden/tensorflow.estimator.-run-config.pbtxt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py index 1820b2b2d4..372f01dc82 100644 --- a/tensorflow/python/estimator/run_config.py +++ b/tensorflow/python/estimator/run_config.py @@ -210,7 +210,7 @@ class RunConfig(object): def __init__(self, model_dir=None, - tf_random_seed=1, + tf_random_seed=None, save_summary_steps=100, save_checkpoints_steps=_USE_DEFAULT, save_checkpoints_secs=_USE_DEFAULT, diff --git a/tensorflow/python/estimator/run_config_test.py b/tensorflow/python/estimator/run_config_test.py index b3c917649f..ecc850d540 100644 --- a/tensorflow/python/estimator/run_config_test.py +++ b/tensorflow/python/estimator/run_config_test.py @@ -70,7 +70,7 @@ class RunConfigTest(test.TestCase): config = run_config_lib.RunConfig() self.assertIsNone(config.model_dir) self.assertIsNone(config.session_config) - self.assertEqual(1, config.tf_random_seed) + self.assertIsNone(config.tf_random_seed) self.assertEqual(100, config.save_summary_steps) self.assertEqual(600, config.save_checkpoints_secs) self.assertIsNone(config.save_checkpoints_steps) diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt index 7ab094c999..d006ecb254 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt @@ -76,7 +76,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'100\', \'\', \'\', \'None\', \'5\', \'10000\', \'100\'], " + argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'\', \'\', \'None\', \'5\', \'10000\', \'100\'], " } member_method { name: "replace" -- GitLab From a925c8dcaf57506c0f7ad167aad6794a88878ca3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 15:03:01 -0700 Subject: [PATCH 075/573] Improve performance of tf.space_to_depth and tf.depth_to_space for typical block sizes of NCHW data layout on GPU with a loop kernel. PiperOrigin-RevId: 172520132 --- .../core/kernels/depthtospace_op_gpu.cu.cc | 79 +++++++++++++++++- .../core/kernels/spacetodepth_op_gpu.cu.cc | 83 +++++++++++++++++-- 2 files changed, 151 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc index 357c1f1be4..7a66285383 100644 --- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc +++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc @@ -67,7 +67,6 @@ __global__ void D2S_NCHW(const int32 nthreads, const int block_size, const int input_width, const int output_depth_by_input_height, dtype* __restrict__ output_ptr) { - // TODO(pauldonnelly): Implement more optimized kernels. CUDA_1D_KERNEL_LOOP(input_idx, nthreads) { // We will be converting the image from ordering: // n, bY, bX, oC, iY, iX (== input_idx) to @@ -99,6 +98,47 @@ __global__ void D2S_NCHW(const int32 nthreads, } } +template +__global__ void D2S_NCHW_LOOP(const int32 nthreads, + const dtype* __restrict__ input, + const int input_width, const int output_width, + const int output_depth_by_input_area, + const int input_depth_by_input_area, + dtype* __restrict__ output) { + CUDA_1D_KERNEL_LOOP(thread_idx, nthreads) { + // We will be converting the image from ordering: + // n, bY, bX, oC, iY, iX to + // n, oC, iY, bY, iX, bX + + // We assume thread_idx encodes n_oC_iY_iX, and use an unrolled loop over + // bY and bX coordinates within the block. This kernel is significantly + // more performant than the D2S_NCHW kernel. + // A likely explanation of the improvement is that although both kernels + // get input coalescing, this one would write the output data more densely + // per warp, so would benefit assuming delayed cache writeback is used. + + const int n_oC_iY = thread_idx / input_width; + const int iX = thread_idx - n_oC_iY * input_width; + + const int n = thread_idx / output_depth_by_input_area; + const int oC_iY_iX = thread_idx - n * output_depth_by_input_area; + + // Recombine the components and apply to the input and output pointers. + auto input_ptr = input + n * input_depth_by_input_area + oC_iY_iX; + auto output_ptr = output + (n_oC_iY * output_width + iX) * block_size; + +#pragma unroll + // Copy a patch of data to the output batch image. + for (int bY = 0; bY < block_size; ++bY) { +#pragma unroll + for (int bX = 0; bX < block_size; ++bX) { + output_ptr[bY * output_width + bX] = ldg( + input_ptr + (bY * block_size + bX) * output_depth_by_input_area); + } + } + } +} + } // namespace // Specialization of DepthToSpaceOpFunctor for a GPUDevice. @@ -139,10 +179,41 @@ struct DepthToSpaceOpFunctor { const int input_height = input.dimension(2); const int input_width = input.dimension(3); const int output_depth = output.dimension(1); - const int total_count = - batch_size * input_height * input_width * input_depth; + const int input_area = input_width * input_height; + const int input_depth_by_input_area = input_depth * input_area; + + // We improve performance by generating instantiations of the loop kernel + // for the most common block sizes. + if (block_size <= 4) { + const int output_width = output.dimension(3); + const int output_depth_by_input_area = output_depth * input_area; + const int total_count = batch_size * output_depth_by_input_area; + CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d); + switch (block_size) { + case 2: + return D2S_NCHW_LOOP + <<>>( + total_count, input.data(), input_width, output_width, + output_depth_by_input_area, input_depth_by_input_area, + output.data()); + case 3: + return D2S_NCHW_LOOP + <<>>( + total_count, input.data(), input_width, output_width, + output_depth_by_input_area, input_depth_by_input_area, + output.data()); + case 4: + return D2S_NCHW_LOOP + <<>>( + total_count, input.data(), input_width, output_width, + output_depth_by_input_area, input_depth_by_input_area, + output.data()); + } + } + + // Other block sizes are processed by the generic kernel. + const int total_count = batch_size * input_depth_by_input_area; auto config = GetCudaLaunchConfig(total_count, d); - D2S_NCHW<<>>( config.virtual_thread_count, input.data(), block_size, input_width, output_depth * input_height, output.data()); diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc index 94c7a0a3f6..a1a01e8813 100644 --- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc +++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc @@ -66,10 +66,6 @@ __global__ void S2D_NCHW(const int32 nthreads, const int block_size, const int output_width, const int input_depth_by_output_height, dtype* __restrict__ output_ptr) { - // TODO(pauldonnelly): This kernel gets input coalescing, but not output - // coalescing. We could use shared memory to get both. It may also help - // to amortize the address calculations via an inner loop over block_size. - // A template parameter for the block_size is another potential optimization. CUDA_1D_KERNEL_LOOP(input_idx, nthreads) { // We assume both the input and output are packed NCHW tensors. // input_idx represents an index within the flattened input tensor. @@ -100,6 +96,48 @@ __global__ void S2D_NCHW(const int32 nthreads, } } +// Space2Depth kernel for FORMAT_NCHW using a loop over block area. +// See 'spacetodepth_op.h' for functional specification. +template +__global__ void S2D_NCHW_LOOP(const int32 nthreads, + const dtype* __restrict__ input, + const int output_width, const int input_width, + const int input_depth_by_output_area, + const int output_depth_by_output_area, + dtype* __restrict__ output) { + CUDA_1D_KERNEL_LOOP(thread_idx, nthreads) { + // We will be converting the image from ordering: + // n, iC, oY, bY, oX, bX (== input index) to + // n, bY, bX, iC, oY, oX (== output index) + + // We assume thread_idx encodes n_iC_oY_oX, and use an unrolled loop over + // bY and bX coordinates within the block. This kernel gets a small + // performance improvement compared with S2D_NCHW due to a denser access + // pattern on the input side. (Note: the equivalent D2S kernel gets a larger + // improvement as a denser pattern on the output side makes more + // difference). + + const int n_iC_oY = thread_idx / output_width; + const int oX = thread_idx - n_iC_oY * output_width; + const int n = thread_idx / input_depth_by_output_area; + const int iC_oY_oX = thread_idx - n * input_depth_by_output_area; + + // Recombine the components and apply to the input and output pointers. + auto input_ptr = input + (n_iC_oY * input_width + oX) * block_size; + auto output_ptr = output + n * output_depth_by_output_area + iC_oY_oX; + +#pragma unroll + // Copy a patch of data to the output batch image. + for (int bY = 0; bY < block_size; ++bY) { +#pragma unroll + for (int bX = 0; bX < block_size; ++bX) { + output_ptr[(bY * block_size + bX) * input_depth_by_output_area] = + ldg(input_ptr + bY * input_width + bX); + } + } + } +} + // Specialization of SpaceToDepthOpFunctor for a CPUDevice. namespace functor { template @@ -137,9 +175,40 @@ struct SpaceToDepthOpFunctor { const int output_depth = output.dimension(1); const int output_height = output.dimension(2); const int output_width = output.dimension(3); - - const int total_count = - batch_size * output_height * output_width * output_depth; + const int output_area = output_width * output_height; + const int output_depth_by_output_area = output_depth * output_area; + + // We improve performance by generating instantiations of the loop kernel + // for the most common block sizes. + if (block_size <= 4) { + const int input_width = input.dimension(3); + const int input_depth_by_output_area = input_depth * output_area; + const int total_count = batch_size * input_depth_by_output_area; + CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d); + switch (block_size) { + case 2: + return S2D_NCHW_LOOP + <<>>( + total_count, input.data(), output_width, input_width, + input_depth_by_output_area, output_depth_by_output_area, + output.data()); + case 3: + return S2D_NCHW_LOOP + <<>>( + total_count, input.data(), output_width, input_width, + input_depth_by_output_area, output_depth_by_output_area, + output.data()); + case 4: + return S2D_NCHW_LOOP + <<>>( + total_count, input.data(), output_width, input_width, + input_depth_by_output_area, output_depth_by_output_area, + output.data()); + } + } + + // Other block sizes are processed by the generic kernel. + const int total_count = batch_size * output_depth_by_output_area; CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d); S2D_NCHW<<>>( config.virtual_thread_count, input.data(), block_size, output_width, -- GitLab From 110dfa8953e7e9b625681744fde51f30ace2aecd Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Tue, 17 Oct 2017 15:11:27 -0700 Subject: [PATCH 076/573] Bugfix: add `fill_triangular`, `reduce_weighted_logsumexp`, `tridiag` to `tf.contrib.distributions.__init__`. PiperOrigin-RevId: 172521590 --- tensorflow/contrib/distributions/__init__.py | 10 +++++++--- .../contrib/distributions/python/ops/mvn_tril.py | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py index f33cc1de0a..16f6533e57 100644 --- a/tensorflow/contrib/distributions/__init__.py +++ b/tensorflow/contrib/distributions/__init__.py @@ -28,8 +28,11 @@ from tensorflow.contrib.distributions.python.ops.chi2 import * from tensorflow.contrib.distributions.python.ops.conditional_distribution import * from tensorflow.contrib.distributions.python.ops.conditional_transformed_distribution import * from tensorflow.contrib.distributions.python.ops.deterministic import * +from tensorflow.contrib.distributions.python.ops.distribution_util import fill_triangular from tensorflow.contrib.distributions.python.ops.distribution_util import matrix_diag_transform +from tensorflow.contrib.distributions.python.ops.distribution_util import reduce_weighted_logsumexp from tensorflow.contrib.distributions.python.ops.distribution_util import softplus_inverse +from tensorflow.contrib.distributions.python.ops.distribution_util import tridiag from tensorflow.contrib.distributions.python.ops.estimator import * from tensorflow.contrib.distributions.python.ops.geometric import * from tensorflow.contrib.distributions.python.ops.independent import * @@ -140,13 +143,14 @@ _allowed_symbols = [ 'RelaxedOneHotCategorical', 'kl_divergence', 'RegisterKL', - 'matrix_diag_transform', 'fill_triangular', + 'matrix_diag_transform', + 'reduce_weighted_logsumexp', + 'softplus_inverse', + 'tridiag', 'normal_conjugates_known_scale_posterior', 'normal_conjugates_known_scale_predictive', - 'softplus_inverse', 'percentile', - 'reduce_weighted_logsumexp', 'assign_moving_mean_variance', 'assign_log_moving_mean_exp', 'moving_mean_variance', diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py index e3d68f6b4c..260dcc18f5 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py @@ -121,6 +121,14 @@ class MultivariateNormalTriL( [-10, 0, 9]] # shape: [2, 3] mvn.prob(x).eval() # shape: [2] + # Instantiate a "learnable" MVN. + dims = 4 + with tf.variable_scope("model"): + mvn = ds.MultivariateNormalTriL( + loc=tf.get_variable(shape=[dims], dtype=tf.float32, name="mu"), + scale_tril=ds.fill_triangular( + tf.get_variable(shape=[dims * (dims + 1) / 2], + dtype=tf.float32, name="chol_Sigma"))) ``` """ -- GitLab From 84edca032719d2aa1da465e26c51d32c205df3bc Mon Sep 17 00:00:00 2001 From: Guangda Lai Date: Tue, 17 Oct 2017 15:18:58 -0700 Subject: [PATCH 077/573] Fix a bug in BFCAllocator::Log2FloorNonZero() under non gnuc non windows mode, where it returns the bit index of the most significant '1' bit, starting from 1, but the expected result should start from 0 (i.e. it should return floor(log2(n)) ). PiperOrigin-RevId: 172522745 --- .../core/common_runtime/bfc_allocator.h | 20 ++++++++----- .../gpu/gpu_bfc_allocator_test.cc | 28 +++++++++++++++---- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/tensorflow/core/common_runtime/bfc_allocator.h b/tensorflow/core/common_runtime/bfc_allocator.h index 326e0ffe40..20fa05f0d2 100644 --- a/tensorflow/core/common_runtime/bfc_allocator.h +++ b/tensorflow/core/common_runtime/bfc_allocator.h @@ -362,6 +362,17 @@ class BFCAllocator : public VisitableAllocator { // Structures immutable after construction size_t memory_limit_ = 0; + + inline int Log2FloorNonZeroSlow(uint64 n) { + int r = 0; + while (n > 0) { + r++; + n >>= 1; + } + return r - 1; + } + + // Returns floor(log2(n)). inline int Log2FloorNonZero(uint64 n) { #if defined(__GNUC__) return 63 ^ __builtin_clzll(n); @@ -370,12 +381,7 @@ class BFCAllocator : public VisitableAllocator { _BitScanReverse64(&index, n); return index; #else - int r = 0; - while (n > 0) { - r++; - n >>= 1; - } - return r; + return Log2FloorNonZeroSlow(n); #endif } @@ -425,7 +431,7 @@ class BFCAllocator : public VisitableAllocator { // Stats. AllocatorStats stats_ GUARDED_BY(lock_); - friend class GPUBFCAllocatorBinDebugInfoTest; + friend class GPUBFCAllocatorPrivateMethodsTest; TF_DISALLOW_COPY_AND_ASSIGN(BFCAllocator); }; diff --git a/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc b/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc index b7554e5b82..00ef130d34 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc @@ -354,12 +354,13 @@ BENCHMARK(BM_AllocationDelayed)->Arg(1)->Arg(10)->Arg(100)->Arg(1000); } // namespace -class GPUBFCAllocatorBinDebugInfoTest : public ::testing::Test { +class GPUBFCAllocatorPrivateMethodsTest : public ::testing::Test { protected: - // This test method is called from a test. The reason for this is that this - // class is a friend class to BFCAllocator, but tests are not, so only this - // method can access the type BFCAllocator::BinDebugInfo. - void testBinDebugInfo() { + // The following test methods are called from tests. The reason for this is + // that this class is a friend class to BFCAllocator, but tests are not, so + // only methods inside this class can access private members of BFCAllocator. + + void TestBinDebugInfo() { GPUBFCAllocator a(0, 1 << 30); std::vector initial_ptrs; @@ -436,9 +437,24 @@ class GPUBFCAllocatorBinDebugInfoTest : public ::testing::Test { } } } + + void TestLog2FloorNonZeroSlow() { + GPUBFCAllocator a(0 /* device_id */, 1 /* total_memory */); + EXPECT_EQ(-1, a.Log2FloorNonZeroSlow(0)); + EXPECT_EQ(0, a.Log2FloorNonZeroSlow(1)); + EXPECT_EQ(1, a.Log2FloorNonZeroSlow(2)); + EXPECT_EQ(1, a.Log2FloorNonZeroSlow(3)); + EXPECT_EQ(9, a.Log2FloorNonZeroSlow(1023)); + EXPECT_EQ(10, a.Log2FloorNonZeroSlow(1024)); + EXPECT_EQ(10, a.Log2FloorNonZeroSlow(1025)); + } }; -TEST_F(GPUBFCAllocatorBinDebugInfoTest, BinDebugInfo) { testBinDebugInfo(); } +TEST_F(GPUBFCAllocatorPrivateMethodsTest, BinDebugInfo) { TestBinDebugInfo(); } + +TEST_F(GPUBFCAllocatorPrivateMethodsTest, Log2FloorNonZeroSlow) { + TestLog2FloorNonZeroSlow(); +} } // namespace tensorflow -- GitLab From 109133639706026d2bd944dd0a0c8fcae0d4fac6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 15:47:49 -0700 Subject: [PATCH 078/573] Add environment variable to enable setting of CUDA context flags. PiperOrigin-RevId: 172527804 --- .../stream_executor/cuda/cuda_platform.cc | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_platform.cc b/tensorflow/stream_executor/cuda/cuda_platform.cc index ee9df5b7de..d69953f557 100644 --- a/tensorflow/stream_executor/cuda/cuda_platform.cc +++ b/tensorflow/stream_executor/cuda/cuda_platform.cc @@ -27,6 +27,43 @@ limitations under the License. namespace perftools { namespace gputools { namespace cuda { +namespace { + +// Synchronize with spinlocks. +const char kScheduleSpinString[] = "spin"; +// Synchronize with spinlocks that also call CPU yield instructions. +const char kScheduleYieldString[] = "yield"; +// Synchronize with a "synchronization primitive" (e.g. mutex). +const char kScheduleBlockingSyncString[] = "blocking_sync"; + +const DeviceOptions GetDeviceOptionsFromEnv() { + const char* gpu_schedule_string = + std::getenv("TF_CUDA_PLATFORM_GPU_DEVICE_SCHEDULE"); + + if (gpu_schedule_string == nullptr) { + return perftools::gputools::DeviceOptions::Default(); + } + + unsigned device_flags = 0; + if (strcasecmp(kScheduleSpinString, gpu_schedule_string) == 0) { + device_flags = perftools::gputools::DeviceOptions::kScheduleSpin; + } else if (strcasecmp(kScheduleYieldString, gpu_schedule_string) == 0) { + device_flags = perftools::gputools::DeviceOptions::kScheduleYield; + } else if (strcasecmp(kScheduleBlockingSyncString, gpu_schedule_string) == + 0) { + device_flags = perftools::gputools::DeviceOptions::kScheduleBlockingSync; + } else { + LOG(QFATAL) << "Unknown option for environment variable " + "TF_CUDA_PLATFORM_GPU_DEVICE_SCHEDULE " + << gpu_schedule_string << " should be one of {" + << kScheduleBlockingSyncString << ", " << kScheduleSpinString + << ", " << kScheduleYieldString << "}"; + } + + return perftools::gputools::DeviceOptions(device_flags); +} + +} // namespace CudaPlatform::CudaPlatform() : name_("CUDA"), min_numa_node_(0), limit_numa_node_(0) {} @@ -112,7 +149,7 @@ port::StatusOr CudaPlatform::ExecutorForDevice(int ordinal) { StreamExecutorConfig config; config.ordinal = ordinal; config.plugin_config = PluginConfig(); - config.device_options = DeviceOptions::Default(); + config.device_options = GetDeviceOptionsFromEnv(); return GetExecutor(config); } @@ -121,7 +158,7 @@ port::StatusOr CudaPlatform::ExecutorForDeviceWithPluginConfig( StreamExecutorConfig config; config.ordinal = device_ordinal; config.plugin_config = plugin_config; - config.device_options = DeviceOptions::Default(); + config.device_options = GetDeviceOptionsFromEnv(); return GetExecutor(config); } -- GitLab From 4c7e02c082fdf3b4b04e42f1880cf6e0ff4fc409 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Tue, 17 Oct 2017 16:05:14 -0700 Subject: [PATCH 079/573] Add bitwise LeftShift (aka. tf.bitwise.left_shift) and RightShift (tf.bitwise_right_shift) operators to Tensorflow. Their semantics are intended to mirror numpy.left_shift and numpy.right_shift. Fix a couple of missing cases of missing uint32/uint64 support exposed when adding tests for bitshifts. PiperOrigin-RevId: 172530375 --- tensorflow/compiler/tests/binary_ops_test.py | 16 +++++++ .../compiler/tf2xla/kernels/binary_ops.cc | 7 +++ tensorflow/contrib/makefile/tf_op_files.txt | 2 + tensorflow/core/framework/types.cc | 12 +++++ tensorflow/core/framework/types.h | 3 ++ tensorflow/core/kernels/BUILD | 2 + .../kernels/cwise_op_gpu_left_shift.cu.cc | 27 +++++++++++ .../kernels/cwise_op_gpu_right_shift.cu.cc | 27 +++++++++++ .../core/kernels/cwise_op_left_shift.cc | 44 ++++++++++++++++++ .../core/kernels/cwise_op_right_shift.cc | 44 ++++++++++++++++++ tensorflow/core/kernels/cwise_ops.h | 46 +++++++++++++++++++ tensorflow/core/ops/bitwise_ops.cc | 33 +++++++++---- .../python/framework/fast_tensor_util.pyx | 12 +++++ tensorflow/python/framework/tensor_util.py | 16 +++++-- tensorflow/python/ops/bitwise_ops.py | 4 ++ tensorflow/python/ops/bitwise_ops_test.py | 42 +++++++++++++++++ .../tools/api/golden/tensorflow.bitwise.pbtxt | 8 ++++ 17 files changed, 334 insertions(+), 11 deletions(-) create mode 100644 tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc create mode 100644 tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc create mode 100644 tensorflow/core/kernels/cwise_op_left_shift.cc create mode 100644 tensorflow/core/kernels/cwise_op_right_shift.cc diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 44b32b1668..b387467246 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -209,6 +209,22 @@ class BinaryOpsTest(XLATestCase): np.array([0b0, 0b101, 0b1001], dtype=dtype), expected=np.array([0b1, 0b101, 0b1001], dtype=dtype)) + lhs = np.array([0, 5, 3, 14], dtype=dtype) + rhs = np.array([5, 0, 7, 11], dtype=dtype) + self._testBinary( + bitwise_ops.left_shift, lhs, rhs, + expected=np.left_shift(lhs, rhs)) + self._testBinary( + bitwise_ops.right_shift, lhs, rhs, + expected=np.right_shift(lhs, rhs)) + + if dtype in [np.int8, np.int16, np.int32, np.int64]: + lhs = np.array([-1, -5, -3, -14], dtype=dtype) + rhs = np.array([5, 0, 1, 11], dtype=dtype) + self._testBinary( + bitwise_ops.right_shift, lhs, rhs, + expected=np.right_shift(lhs, rhs)) + def testNumericOps(self): for dtype in self.numeric_types: self._testBinary( diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc index d635507989..4673bbda14 100644 --- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc @@ -98,6 +98,13 @@ XLA_MAKE_BINARY(FloorMod, XLA_MAKE_BINARY(BitwiseAnd, b->And(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(BitwiseOr, b->Or(lhs, rhs, extend_dimensions)); + +XLA_MAKE_BINARY(LeftShift, b->ShiftLeft(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(RightShift, + (DataTypeIsUnsigned(ctx->input_type(0)) + ? b->ShiftRightLogical(lhs, rhs, extend_dimensions) + : b->ShiftRightArithmetic(lhs, rhs, extend_dimensions))); + XLA_MAKE_BINARY(LogicalAnd, b->And(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(LogicalOr, b->Or(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Mod, b->Rem(lhs, rhs, extend_dimensions)); diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt index 1fda907074..a8690a04ad 100644 --- a/tensorflow/contrib/makefile/tf_op_files.txt +++ b/tensorflow/contrib/makefile/tf_op_files.txt @@ -170,6 +170,8 @@ tensorflow/core/kernels/cwise_op_div.cc tensorflow/core/kernels/cwise_op_bitwise_xor.cc tensorflow/core/kernels/cwise_op_bitwise_or.cc tensorflow/core/kernels/cwise_op_bitwise_and.cc +tensorflow/core/kernels/cwise_op_left_shift.cc +tensorflow/core/kernels/cwise_op_right_shift.cc tensorflow/core/kernels/cwise_op_add_2.cc tensorflow/core/kernels/cwise_op_add_1.cc tensorflow/core/kernels/cwise_op_abs.cc diff --git a/tensorflow/core/framework/types.cc b/tensorflow/core/framework/types.cc index cc86871cae..faae19585d 100644 --- a/tensorflow/core/framework/types.cc +++ b/tensorflow/core/framework/types.cc @@ -335,6 +335,18 @@ bool DataTypeIsInteger(DataType dt) { } } +bool DataTypeIsUnsigned(DataType dt) { + switch (dt) { + case DT_UINT8: + case DT_UINT16: + case DT_UINT32: + case DT_UINT64: + return true; + default: + return false; + } +} + int DataTypeSize(DataType dt) { #define CASE(T) \ case DataTypeToEnum::value: \ diff --git a/tensorflow/core/framework/types.h b/tensorflow/core/framework/types.h index 300a57e948..dc53ed4178 100644 --- a/tensorflow/core/framework/types.h +++ b/tensorflow/core/framework/types.h @@ -227,6 +227,9 @@ bool DataTypeIsQuantized(DataType dt); // Is the dtype nonquantized integral? bool DataTypeIsInteger(DataType dt); +// Is the dtype an unsigned integral type? +bool DataTypeIsUnsigned(DataType dt); + // Returns a 0 on failure int DataTypeSize(DataType dt); diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index d1a2362e5e..d931f12f6d 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -4512,6 +4512,7 @@ filegroup( "cwise_op_greater_equal.cc", "cwise_op_invert.cc", "cwise_op_isfinite.cc", + "cwise_op_left_shift.cc", "cwise_op_less.cc", "cwise_op_less_equal.cc", "cwise_op_log.cc", @@ -4525,6 +4526,7 @@ filegroup( "cwise_op_neg.cc", "cwise_op_pow.cc", "cwise_op_reciprocal.cc", + "cwise_op_right_shift.cc", "cwise_op_round.cc", "cwise_op_rsqrt.cc", "cwise_op_select.cc", diff --git a/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc new file mode 100644 index 0000000000..740048795a --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc @@ -0,0 +1,27 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h" + +namespace tensorflow { +namespace functor { +DEFINE_BINARY8(left_shift, int8, int16, int32, int64, uint8, uint16, uint32, + uint64); +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc new file mode 100644 index 0000000000..bb6772772c --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc @@ -0,0 +1,27 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h" + +namespace tensorflow { +namespace functor { +DEFINE_BINARY8(right_shift, int8, int16, int32, int64, uint8, uint16, uint32, + uint64); +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/cwise_op_left_shift.cc b/tensorflow/core/kernels/cwise_op_left_shift.cc new file mode 100644 index 0000000000..ccb68139de --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_left_shift.cc @@ -0,0 +1,44 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/kernels/cwise_ops_common.h" + +namespace tensorflow { +REGISTER8(BinaryOp, CPU, "LeftShift", functor::left_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); + +#if TENSORFLOW_USE_SYCL +#define REGISTER_SYCL_KERNEL(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("LeftShift").Device(DEVICE_SYCL).TypeConstraint("T"), \ + BinaryOp>); +REGISTER_SYCL_KERNEL(int8); +REGISTER_SYCL_KERNEL(int16); +REGISTER_SYCL_KERNEL(int32); +REGISTER_SYCL_KERNEL(int64); +REGISTER_SYCL_KERNEL(uint8); +REGISTER_SYCL_KERNEL(uint16); +REGISTER_SYCL_KERNEL(uint32); +REGISTER_SYCL_KERNEL(uint64); +#undef REGISTER_SYCL_KERNEL + +#endif // TENSORFLOW_USE_SYCL + +#if GOOGLE_CUDA +REGISTER8(BinaryOp, GPU, "LeftShift", functor::left_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); +#endif // GOOGLE_CUDA + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_right_shift.cc b/tensorflow/core/kernels/cwise_op_right_shift.cc new file mode 100644 index 0000000000..6dc6b97e35 --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_right_shift.cc @@ -0,0 +1,44 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/kernels/cwise_ops_common.h" + +namespace tensorflow { +REGISTER8(BinaryOp, CPU, "RightShift", functor::right_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); + +#if TENSORFLOW_USE_SYCL +#define REGISTER_SYCL_KERNEL(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("RightShift").Device(DEVICE_SYCL).TypeConstraint("T"), \ + BinaryOp>); +REGISTER_SYCL_KERNEL(int8); +REGISTER_SYCL_KERNEL(int16); +REGISTER_SYCL_KERNEL(int32); +REGISTER_SYCL_KERNEL(int64); +REGISTER_SYCL_KERNEL(uint8); +REGISTER_SYCL_KERNEL(uint16); +REGISTER_SYCL_KERNEL(uint32); +REGISTER_SYCL_KERNEL(uint64); +#undef REGISTER_SYCL_KERNEL + +#endif // TENSORFLOW_USE_SYCL + +#if GOOGLE_CUDA +REGISTER8(BinaryOp, GPU, "RightShift", functor::right_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); +#endif // GOOGLE_CUDA + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h index d935331904..89487419ee 100644 --- a/tensorflow/core/kernels/cwise_ops.h +++ b/tensorflow/core/kernels/cwise_ops.h @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/tensor_types.h" @@ -810,6 +812,50 @@ struct bitwise_or : base> {}; template struct bitwise_xor : base> {}; +template +struct left_shift_op { + EIGEN_EMPTY_STRUCT_CTOR(left_shift_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x, + const T& y) const { + // Avoids UB: don't shift by larger than the bitwidth of T, and + // performs left shifts as unsigned shifts. + T y_clamped = y; + if (y_clamped < 0) { + y_clamped = 0; + } else if (y_clamped > sizeof(T) * CHAR_BIT - 1) { + y_clamped = sizeof(T) * CHAR_BIT - 1; + } + using U = typename std::make_unsigned::type; + return static_cast(static_cast(x) << static_cast(y_clamped)); + } +}; + +template +struct right_shift_op { + EIGEN_EMPTY_STRUCT_CTOR(right_shift_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x, + const T& y) const { + // Avoids UB: don't shift by larger than the bitwidth of T. + T y_clamped = y; + if (y_clamped < 0) { + y_clamped = 0; + } else if (y_clamped > sizeof(T) * CHAR_BIT - 1) { + y_clamped = sizeof(T) * CHAR_BIT - 1; + } + // Technically right shifts of signed integers are not necessarily + // arithmetic shifts according to the C++ standard. However in practice most + // implementations are arithmetic shifts. If this proves to be a problem in + // practice, we may need to use an alternative implementation. + return x >> y_clamped; + } +}; + +template +struct left_shift : base> {}; + +template +struct right_shift : base> {}; + template struct make_complex_func { typedef std::complex result_type; diff --git a/tensorflow/core/ops/bitwise_ops.cc b/tensorflow/core/ops/bitwise_ops.cc index 3ffc4ab74a..3156162b78 100644 --- a/tensorflow/core/ops/bitwise_ops.cc +++ b/tensorflow/core/ops/bitwise_ops.cc @@ -23,7 +23,7 @@ namespace tensorflow { REGISTER_OP("Invert") .Input("x: T") .Output("y: T") - .Attr("T: {int8, int16, int32, int64, uint8, uint16}") + .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Flips all bits elementwise. @@ -32,18 +32,18 @@ The result will have exactly those bits set, that are not set in `x`. The computation is performed on the underlying representation of x. )doc"); -#define BINARY_BITWISE() \ - Input("x: T") \ - .Input("y: T") \ - .Output("z: T") \ - .SetIsCommutative() \ - .Attr("T: {int8, int16, int32, int64, uint8, uint16}") \ +#define BINARY_BITWISE() \ + Input("x: T") \ + .Input("y: T") \ + .Output("z: T") \ + .SetIsCommutative() \ + .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") \ .SetShapeFn(shape_inference::UnchangedShape) REGISTER_OP("PopulationCount") .Input("x: T") .Output("y: uint8") - .Attr("T: {int8, int16, int32, int64, uint8, uint16}") + .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Computes element-wise population count (a.k.a. popcount, bitsum, bitcount). @@ -77,4 +77,21 @@ The result will have those bits set, that are different in `x` and `y`. The computation is performed on the underlying representations of `x` and `y`. )doc"); +REGISTER_OP("LeftShift").BINARY_BITWISE().Doc(R"doc( +Elementwise computes the bitwise left-shift of `x` and `y`. + +If `y` is negative, or greater than or equal to the width of `x` in bits the +result is implementation defined. +)doc"); + +REGISTER_OP("RightShift").BINARY_BITWISE().Doc(R"doc( +Elementwise computes the bitwise right-shift of `x` and `y`. + +Performs a logical shift for unsigned integer types, and an arithmetic shift +for signed integer types. + +If `y` is negative, or greater than or equal to than the width of `x` in bits +the result is implementation defined. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx index b43ddb4ad3..19928314ef 100644 --- a/tensorflow/python/framework/fast_tensor_util.pyx +++ b/tensorflow/python/framework/fast_tensor_util.pyx @@ -30,6 +30,12 @@ def AppendInt32ArrayToTensorProto( for i in range(n): tensor_proto.int_val.append(nparray[i]) +def AppendUInt32ArrayToTensorProto( + tensor_proto, np.ndarray[np.uint32_t, ndim=1] nparray): + cdef long i, n + n = nparray.size + for i in range(n): + tensor_proto.uint32_val.append(nparray[i]) def AppendInt64ArrayToTensorProto( tensor_proto, np.ndarray[np.int64_t, ndim=1] nparray): @@ -38,6 +44,12 @@ def AppendInt64ArrayToTensorProto( for i in range(n): tensor_proto.int64_val.append(nparray[i]) +def AppendUInt64ArrayToTensorProto( + tensor_proto, np.ndarray[np.uint64_t, ndim=1] nparray): + cdef long i, n + n = nparray.size + for i in range(n): + tensor_proto.uint64_val.append(nparray[i]) def AppendUInt8ArrayToTensorProto( tensor_proto, np.ndarray[np.uint8_t, ndim=1] nparray): diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py index 53eba8b747..7e74c19124 100644 --- a/tensorflow/python/framework/tensor_util.py +++ b/tensorflow/python/framework/tensor_util.py @@ -61,6 +61,8 @@ if _FAST_TENSOR_UTIL_AVAILABLE: np.int64: fast_tensor_util.AppendInt64ArrayToTensorProto, np.uint8: fast_tensor_util.AppendUInt8ArrayToTensorProto, np.uint16: fast_tensor_util.AppendUInt16ArrayToTensorProto, + np.uint32: fast_tensor_util.AppendUInt32ArrayToTensorProto, + np.uint64: fast_tensor_util.AppendUInt64ArrayToTensorProto, np.int8: fast_tensor_util.AppendInt8ArrayToTensorProto, np.int16: fast_tensor_util.AppendInt16ArrayToTensorProto, np.complex64: fast_tensor_util.AppendComplex64ArrayToTensorProto, @@ -90,11 +92,17 @@ else: def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values): tensor_proto.int_val.extend([np.asscalar(x) for x in proto_values]) + def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values]) + def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values): tensor_proto.int_val.extend([np.asscalar(x[0]) for x in proto_values]) - def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values): - tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values]) + def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.uint32_val.extend([np.asscalar(x) for x in proto_values]) + + def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.uint64_val.extend([np.asscalar(x) for x in proto_values]) def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values): tensor_proto.scomplex_val.extend([np.asscalar(v) @@ -120,6 +128,8 @@ else: np.int64: SlowAppendInt64ArrayToTensorProto, np.uint8: SlowAppendIntArrayToTensorProto, np.uint16: SlowAppendIntArrayToTensorProto, + np.uint32: SlowAppendUInt32ArrayToTensorProto, + np.uint64: SlowAppendUInt64ArrayToTensorProto, np.int8: SlowAppendIntArrayToTensorProto, np.int16: SlowAppendIntArrayToTensorProto, np.complex64: SlowAppendComplex64ArrayToTensorProto, @@ -190,7 +200,7 @@ def _FlattenToStrings(nested_strings): _TENSOR_CONTENT_TYPES = frozenset([ dtypes.float32, dtypes.float64, dtypes.int32, dtypes.uint8, dtypes.int16, dtypes.int8, dtypes.int64, dtypes.qint8, dtypes.quint8, dtypes.qint16, - dtypes.quint16, dtypes.qint32, + dtypes.quint16, dtypes.qint32, dtypes.uint32, dtypes.uint64 ]) diff --git a/tensorflow/python/ops/bitwise_ops.py b/tensorflow/python/ops/bitwise_ops.py index 44daf13537..e8e187e68f 100644 --- a/tensorflow/python/ops/bitwise_ops.py +++ b/tensorflow/python/ops/bitwise_ops.py @@ -19,6 +19,8 @@ @@bitwise_or @@bitwise_xor @@invert +@@left_shift +@@right_shift """ from __future__ import absolute_import @@ -37,5 +39,7 @@ ops.NotDifferentiable("BitwiseOr") ops.NotDifferentiable("BitwiseXor") ops.NotDifferentiable("Invert") ops.NotDifferentiable("PopulationCount") +ops.NotDifferentiable("LeftShift") +ops.NotDifferentiable("RightShift") remove_undocumented(__name__) diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py index 1d08c8f82d..fa1b219b17 100644 --- a/tensorflow/python/ops/bitwise_ops_test.py +++ b/tensorflow/python/ops/bitwise_ops_test.py @@ -93,5 +93,47 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): expected = [dtype.max - x for x in inputs] self.assertAllEqual(inverted, expected) + def testShiftsWithPositiveLHS(self): + dtype_list = [np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64] + + with self.test_session(use_gpu=True) as sess: + for dtype in dtype_list: + lhs = np.array([0, 5, 3, 14], dtype=dtype) + rhs = np.array([5, 0, 7, 3], dtype=dtype) + left_shift_result, right_shift_result = sess.run( + [bitwise_ops.left_shift(lhs, rhs), + bitwise_ops.right_shift(lhs, rhs)]) + self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs)) + self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs)) + + def testShiftsWithNegativeLHS(self): + dtype_list = [np.int8, np.int16, np.int32, np.int64] + + with self.test_session(use_gpu=True) as sess: + for dtype in dtype_list: + lhs = np.array([-1, -5, -3, -14], dtype=dtype) + rhs = np.array([5, 0, 7, 11], dtype=dtype) + left_shift_result, right_shift_result = sess.run( + [bitwise_ops.left_shift(lhs, rhs), + bitwise_ops.right_shift(lhs, rhs)]) + self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs)) + self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs)) + + def testImplementationDefinedShiftsDoNotCrash(self): + dtype_list = [np.int8, np.int16, np.int32, np.int64] + + with self.test_session(use_gpu=True) as sess: + for dtype in dtype_list: + lhs = np.array([-1, -5, -3, -14], dtype=dtype) + rhs = np.array([-2, 64, 101, 32], dtype=dtype) + # We intentionally do not test for specific values here since the exact + # outputs are implementation-defined. However, we should not crash or + # trigger an undefined-behavior error from tools such as + # AddressSanitizer. + sess.run([bitwise_ops.left_shift(lhs, rhs), + bitwise_ops.right_shift(lhs, rhs)]) + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt b/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt index 1e4d333cc0..01cbd55c5d 100644 --- a/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt @@ -16,4 +16,12 @@ tf_module { name: "invert" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "left_shift" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "right_shift" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } } -- GitLab From 246e66c01cbf515174673c766f81705380ab69f6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 16:08:07 -0700 Subject: [PATCH 080/573] Update ops-related pbtxt files. PiperOrigin-RevId: 172530775 --- .../core/ops/compat/ops_history.v1.pbtxt | 214 ++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 78 +++++++ 2 files changed, 292 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index d93a4ff933..6772024263 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -5733,6 +5733,68 @@ op { } is_commutative: true } +op { + name: "BitwiseAnd" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} +op { + name: "BitwiseOr" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + } + } + } + is_commutative: true +} op { name: "BitwiseOr" input_arg { @@ -5758,6 +5820,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -5793,6 +5857,38 @@ op { } is_commutative: true } +op { + name: "BitwiseXor" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} op { name: "BroadcastArgs" input_arg { @@ -14315,6 +14411,33 @@ op { } } } +op { + name: "Invert" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } +} op { name: "InvertPermutation" input_arg { @@ -14818,6 +14941,38 @@ op { } is_stateful: true } +op { + name: "LeftShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} op { name: "Less" input_arg { @@ -20883,6 +21038,33 @@ op { } } } +op { + name: "PopulationCount" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type: DT_UINT8 + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } +} op { name: "Pow" input_arg { @@ -28983,6 +29165,38 @@ op { } } } +op { + name: "RightShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} op { name: "Rint" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 6403dcf78c..623f5457bb 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -4008,6 +4008,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -4040,6 +4042,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -4072,6 +4076,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -11083,6 +11089,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -11564,6 +11572,40 @@ op { description: "See explanations of candidate sampling and the data formats at\ngo/candidate-sampling.\n\nFor each batch, this op picks a single set of sampled candidate labels.\n\nThe advantages of sampling candidates per-batch are simplicity and the\npossibility of efficient dense matrix multiplication. The disadvantage is that\nthe sampled candidates must be chosen independently of the context and of the\ntrue labels." is_stateful: true } +op { + name: "LeftShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + summary: "Elementwise computes the bitwise left-shift of `x` and `y`." + description: "If `y` is negative, or greater than or equal to the width of `x` in bits the\nresult is implementation defined." + is_commutative: true +} op { name: "Less" input_arg { @@ -16522,6 +16564,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -23211,6 +23255,40 @@ op { summary: "Reverses specific dimensions of a tensor." description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0, 1, 2, 3],\n# [ 4, 5, 6, 7],\n# [ 8, 9, 10, 11]],\n# [[12, 13, 14, 15],\n# [16, 17, 18, 19],\n# [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is -1\nreverse(t, dims) ==> [[[[ 3, 2, 1, 0],\n [ 7, 6, 5, 4],\n [ 11, 10, 9, 8]],\n [[15, 14, 13, 12],\n [19, 18, 17, 16],\n [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n [16, 17, 18, 19],\n [20, 21, 22, 23]\n [[ 0, 1, 2, 3],\n [ 4, 5, 6, 7],\n [ 8, 9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n [4, 5, 6, 7],\n [0, 1, 2, 3]]\n [[20, 21, 22, 23],\n [16, 17, 18, 19],\n [12, 13, 14, 15]]]]\n```" } +op { + name: "RightShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + summary: "Elementwise computes the bitwise right-shift of `x` and `y`." + description: "Performs a logical shift for unsigned integer types, and an arithmetic shift\nfor signed integer types.\n\nIf `y` is negative, or greater than or equal to than the width of `x` in bits\nthe result is implementation defined." + is_commutative: true +} op { name: "Rint" input_arg { -- GitLab From 95c7f5344f8da74a839c459c6415855bffe4f004 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 16:15:29 -0700 Subject: [PATCH 081/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 172531834 --- tensorflow/go/op/wrappers.go | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 8f5ee9c3df..c117711c81 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -2324,6 +2324,45 @@ func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (aud return op.Output(0), op.Output(1) } +// Elementwise computes the bitwise right-shift of `x` and `y`. +// +// Performs a logical shift for unsigned integer types, and an arithmetic shift +// for signed integer types. +// +// If `y` is negative, or greater than or equal to than the width of `x` in bits +// the result is implementation defined. +func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "RightShift", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Elementwise computes the bitwise left-shift of `x` and `y`. +// +// If `y` is negative, or greater than or equal to the width of `x` in bits the +// result is implementation defined. +func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "LeftShift", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Elementwise computes the bitwise AND of `x` and `y`. // // The result will have those bits set, that are set in both `x` and `y`. The -- GitLab From 47e4d4b6b5742350233a8fd83cd81269792ed286 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 16:19:08 -0700 Subject: [PATCH 082/573] Use optimized functor for conjugate transpose in MatrixInverseOp. Introduce convenience functions DoMatrixTranspose and DoConjugateMatrixTranspose. Misc. minor cleanup of templates in transpose_functor*. PiperOrigin-RevId: 172532252 --- tensorflow/core/kernels/cuda_solvers.h | 8 - .../core/kernels/cuda_solvers_gpu.cu.cc | 18 --- tensorflow/core/kernels/matrix_inverse_op.cc | 12 +- tensorflow/core/kernels/matrix_solve_op.cc | 9 +- tensorflow/core/kernels/qr_op_impl.h | 9 +- .../kernels/self_adjoint_eig_v2_op_gpu.cc | 5 +- tensorflow/core/kernels/svd_op_gpu.cu.cc | 25 ++- tensorflow/core/kernels/transpose_functor.h | 150 ++++++++++-------- .../core/kernels/transpose_functor_cpu.cc | 72 +++++---- .../core/kernels/transpose_functor_gpu.cu.cc | 52 +++--- 10 files changed, 174 insertions(+), 186 deletions(-) diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h index 60c4a0bfb4..eb720b191f 100644 --- a/tensorflow/core/kernels/cuda_solvers.h +++ b/tensorflow/core/kernels/cuda_solvers.h @@ -409,14 +409,6 @@ class DeviceLapackInfo : public ScratchSpace { }; namespace functor { -// Helper functor to transpose and conjugate all matrices in a flattened batch. -template -struct AdjointBatchFunctor { - // We assume that the tensor sizes are correct. - void operator()(const Device& device, - typename TTypes::ConstTensor input, - typename TTypes::Tensor output); -}; // Helper functor to compute the product of diagonal elements in all matrices // in a flattened batch. diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc index 79961c01ca..4171f9d68e 100644 --- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc +++ b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc @@ -29,24 +29,6 @@ namespace functor { typedef Eigen::GpuDevice GPUDevice; -// TODO(rmlarsen): Add a faster custom kernel similar to -// SwapDimension1And2InTensor3 in tensorflow/core/kernels/conv_ops_gpu_3.cu.cc -template -struct AdjointBatchFunctor { - void operator()(const GPUDevice& device, - typename TTypes::ConstTensor input, - typename TTypes::Tensor output) { - const Eigen::array perm({0, 2, 1}); - To32Bit(output).device(device) = To32Bit(input).shuffle(perm).conjugate(); - } -}; - -// Instantiate implementations for the 4 numeric types. -template struct AdjointBatchFunctor; -template struct AdjointBatchFunctor; -template struct AdjointBatchFunctor; -template struct AdjointBatchFunctor; - namespace { // Hacks around missing support for complex arithmetic in nvcc. diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc index 832e508bb7..64edfe470d 100644 --- a/tensorflow/core/kernels/matrix_inverse_op.cc +++ b/tensorflow/core/kernels/matrix_inverse_op.cc @@ -33,6 +33,7 @@ limitations under the License. #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/kernels/transpose_functor.h" #endif namespace tensorflow { @@ -135,15 +136,15 @@ class MatrixInverseOpGpu : public AsyncOpKernel { input.shape(), &input_copy), done); auto input_copy_reshaped = input_copy.template flat_inner_dims(); - auto input_reshaped = input.template flat_inner_dims(); const GPUDevice& device = context->eigen_device(); if (!adjoint_) { device.memcpy(input_copy.flat().data(), input.flat().data(), input.NumElements() * sizeof(Scalar)); } else { - functor::AdjointBatchFunctor functor; - functor(device, input_reshaped, input_copy_reshaped); + OP_REQUIRES_OK_ASYNC( + context, DoConjugateMatrixTranspose(device, input, &input_copy), + done); } const int64 batch_size = input_copy_reshaped.dimension(0); @@ -238,10 +239,7 @@ class MatrixInverseOpGpu : public AsyncOpKernel { done); } } - // Callback for checking info after kernels finish. Also capture the - // temporary Tensors/ScratchSpace so they don't get deallocated before the - // kernels run. TODO(rmlarsen): Use move capture once C++14 becomes - // available. + // Callback for checking info after kernels finish. auto info_checker = [context, done]( const Status& status, const std::vector& host_infos) { diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc index 862033e9fa..2e4098dfab 100644 --- a/tensorflow/core/kernels/matrix_solve_op.cc +++ b/tensorflow/core/kernels/matrix_solve_op.cc @@ -181,9 +181,6 @@ class MatrixSolveOpGpu : public AsyncOpKernel { // false, try to reuse the input buffer if this op owns it exclusively. Tensor input_copy; const GPUDevice& device = context->eigen_device(); - std::vector perm(ndims); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[ndims - 2], perm[ndims - 1]); if (adjoint_) { // For the adjoint case, it is simpler to always make a transposed copy up // front. @@ -193,7 +190,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel { input.shape(), &input_copy), done); OP_REQUIRES_OK_ASYNC(context, - DoTranspose(device, input, perm, &input_copy), done); + DoMatrixTranspose(device, input, &input_copy), done); } else { OP_REQUIRES_OK_ASYNC( context, @@ -267,7 +264,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel { done); if (nrhs > 1) { OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, rhs, perm, &transposed_rhs), done); + context, DoMatrixTranspose(device, rhs, &transposed_rhs), done); } else { device.memcpy(transposed_rhs.flat().data(), rhs.flat().data(), @@ -327,7 +324,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel { // 4. Transpose X to get the final result in row-major form. if (nrhs > 1) { OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, transposed_rhs, perm, output), done); + context, DoMatrixTranspose(device, transposed_rhs, output), done); } else { device.memcpy(output->flat().data(), transposed_rhs.flat().data(), diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/qr_op_impl.h index e263eb22f1..c51d601437 100644 --- a/tensorflow/core/kernels/qr_op_impl.h +++ b/tensorflow/core/kernels/qr_op_impl.h @@ -190,12 +190,9 @@ class QrOpGpu : public AsyncOpKernel { // Transpose input, since cuSolver uses column-major, while TensorFlow uses // row-major storage. - std::vector perm(ndims); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[ndims - 2], perm[ndims - 1]); const GPUDevice& device = context->eigen_device(); OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input, perm, &input_transposed), done); + context, DoMatrixTranspose(device, input, &input_transposed), done); // Compute QR decomposition in-place in input_transposed. std::vector dev_info; @@ -218,7 +215,7 @@ class QrOpGpu : public AsyncOpKernel { // and copy it to the output buffer. if (full_matrices_ || m == n) { OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input_transposed, perm, r), done); + context, DoMatrixTranspose(device, input_transposed, r), done); } else { const Scalar alpha(1); const Scalar beta(0); @@ -280,7 +277,7 @@ class QrOpGpu : public AsyncOpKernel { done); } OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input_transposed, perm, q), done); + context, DoMatrixTranspose(device, input_transposed, q), done); } // Asynchronously check return status from cuSolver kernels. diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc b/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc index b0b4f89a27..3a84df07a9 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc +++ b/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc @@ -148,11 +148,8 @@ class SelfAdjointEigV2OpGpu : public AsyncOpKernel { if (compute_v_) { // Transpose eigenvectors now stored in input_copy in column-major form to // output in row-major form. - std::vector perm(ndims); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[ndims - 2], perm[ndims - 1]); OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input_copy, perm, eigenvectors), done); + context, DoMatrixTranspose(device, input_copy, eigenvectors), done); } // Asynchronously check return status from cuSolver kernels. diff --git a/tensorflow/core/kernels/svd_op_gpu.cu.cc b/tensorflow/core/kernels/svd_op_gpu.cu.cc index 1603a8aeda..dedc2da60b 100644 --- a/tensorflow/core/kernels/svd_op_gpu.cu.cc +++ b/tensorflow/core/kernels/svd_op_gpu.cu.cc @@ -190,8 +190,8 @@ class SvdOpGpu : public AsyncOpKernel { // TODO: can the two cases (MgeqN and MlessN) be simplified, // common boilerplate be reduced, or even combined in one method? void PerformSVD_MgeqN(OpKernelContext* context, DoneCallback done, int64 m, - int64 n, int64 p, const gtl::ArraySlice& perm, - const Tensor& M, Tensor* S, Tensor* U, Tensor* V) { + int64 n, int64 p, const Tensor& M, Tensor* S, Tensor* U, + Tensor* V) { TensorShape shapeRaw = M.shape(); shapeRaw.RemoveLastDims(2); @@ -207,7 +207,7 @@ class SvdOpGpu : public AsyncOpKernel { solver->allocate_scoped_tensor(M.dtype(), input_shape, &input_copy), done); auto device = context->eigen_device(); - OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, M, perm, &input_copy), + OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, M, &input_copy), done); // I need to transpose U at the end @@ -250,7 +250,7 @@ class SvdOpGpu : public AsyncOpKernel { // Transpose U if (compute_uv_) { - OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, u_copy, perm, U), done); + OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, u_copy, U), done); } // now check if the SVD operation succeeded or not @@ -259,8 +259,8 @@ class SvdOpGpu : public AsyncOpKernel { // The SVD if m < n void PerformSVD_MlessN(OpKernelContext* context, DoneCallback done, int64 m, - int64 n, int64 p, const gtl::ArraySlice& perm, - const Tensor& M, Tensor* S, Tensor* U, Tensor* V) { + int64 n, int64 p, const Tensor& M, Tensor* S, + Tensor* U, Tensor* V) { // Perform the SVD on M' // Reuse the input buffer or make a copy for the SVD depending on whether @@ -325,7 +325,7 @@ class SvdOpGpu : public AsyncOpKernel { // Transpose V if (compute_uv_) { auto device = context->eigen_device(); - OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, v_copy, perm, V), done); + OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, v_copy, V), done); } // now check if the SVD operation succeeded or not @@ -389,19 +389,12 @@ class SvdOpGpu : public AsyncOpKernel { return; } - // Prepare permutation - std::vector perm; - for (size_t i = 0; i < ndims - 2; ++i) perm.push_back(i); - perm.push_back(ndims - 1); // transpose last two dimensions - perm.push_back(ndims - 2); - gtl::ArraySlice permAS(perm); - // call implementations if (m >= n) { - PerformSVD_MgeqN(context, done, m, n, p, permAS, input, outputS, outputU, + PerformSVD_MgeqN(context, done, m, n, p, input, outputS, outputU, outputV); } else { - PerformSVD_MlessN(context, done, m, n, p, permAS, input, outputS, outputU, + PerformSVD_MlessN(context, done, m, n, p, input, outputS, outputU, outputV); } } diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h index 87569f0275..a2eb0263e8 100644 --- a/tensorflow/core/kernels/transpose_functor.h +++ b/tensorflow/core/kernels/transpose_functor.h @@ -23,7 +23,6 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" namespace tensorflow { - // Transpose tensor 'in' into tensor 'out' according to dimension // permutation 'perm'. // @@ -46,6 +45,17 @@ template Status DoConjugateTranspose(const Device& device, const Tensor& in, const gtl::ArraySlice perm, Tensor* out); +// Convenience versions of DoTranspose that only swap the last (inner) two +// dimensions. +template +Status DoMatrixTranspose(const Device& device, const Tensor& in, Tensor* out); + +// Convenience versions of DoConjugateTranspose that only swap the last (inner) +// two dimensions. +template +Status DoConjugateMatrixTranspose(const Device& device, const Tensor& in, + Tensor* out); + // Primary device specific functor to be specialized for each device and type. template struct Transpose { @@ -131,11 +141,6 @@ inline bool NonSingletonDimensionsAlign(const TensorShape& input_shape, return true; } -// Device-specific naive implementation for transpose. -template -void TransposeSimple(const Device& d, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out); - // Uses Eigen to transpose. template void TransposeUsingEigen(const Device& d, const Tensor& in, @@ -157,69 +162,78 @@ void TransposeUsingEigen(const Device& d, const Tensor& in, } template -struct DoTransposeImpl { - static Status run(const Device& d, const Tensor& in, - const gtl::ArraySlice perm, bool conjugate, - Tensor* out) { - CHECK_GE(in.dims(), 2); - CHECK_EQ(in.dims(), out->dims()); - CHECK_EQ(in.dims(), perm.size()); - CHECK_EQ(in.dtype(), out->dtype()); - switch (in.dtype()) { - case DT_BOOL: - case DT_INT8: - case DT_QINT8: - case DT_QUINT8: - case DT_UINT8: - Transpose::run(d, in, perm, out); - break; - - case DT_BFLOAT16: - case DT_HALF: - case DT_INT16: - case DT_QINT16: - case DT_QUINT16: - case DT_UINT16: - Transpose::run(d, in, perm, out); - break; - - case DT_FLOAT: - case DT_INT32: - case DT_QINT32: - Transpose::run(d, in, perm, out); - break; - - case DT_DOUBLE: - case DT_INT64: - Transpose::run(d, in, perm, out); - break; - - case DT_COMPLEX64: - if (conjugate) { - Transpose::run(d, in, perm, out); - } else { - Transpose::run(d, in, perm, out); - } - break; - - case DT_COMPLEX128: - if (conjugate) { - Transpose::run(d, in, perm, out); - } else { - Transpose::run(d, in, perm, out); - } - break; - - case DT_STRING: - Transpose::run(d, in, perm, out); - break; - - default: - return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype()); - } - return Status::OK(); +Status DoTransposeImpl(const Device& d, const Tensor& in, + const gtl::ArraySlice perm, bool conjugate, + Tensor* out) { + CHECK_GE(in.dims(), 2); + CHECK_EQ(in.dims(), out->dims()); + CHECK_EQ(in.dims(), perm.size()); + CHECK_EQ(in.dtype(), out->dtype()); + switch (in.dtype()) { + case DT_BOOL: + case DT_INT8: + case DT_QINT8: + case DT_QUINT8: + case DT_UINT8: + Transpose::run(d, in, perm, out); + break; + + case DT_BFLOAT16: + case DT_HALF: + case DT_INT16: + case DT_QINT16: + case DT_QUINT16: + case DT_UINT16: + Transpose::run(d, in, perm, out); + break; + + case DT_FLOAT: + case DT_INT32: + case DT_QINT32: + Transpose::run(d, in, perm, out); + break; + + case DT_DOUBLE: + case DT_INT64: + Transpose::run(d, in, perm, out); + break; + + case DT_COMPLEX64: + if (conjugate) { + Transpose::run(d, in, perm, out); + } else { + Transpose::run(d, in, perm, out); + } + break; + + case DT_COMPLEX128: + if (conjugate) { + Transpose::run(d, in, perm, out); + } else { + Transpose::run(d, in, perm, out); + } + break; + + case DT_STRING: + Transpose::run(d, in, perm, out); + break; + + default: + return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype()); } -}; + return Status::OK(); +} + +template +inline Status DoMatrixTransposeImpl(const Device& device, const Tensor& in, + bool conjugate, Tensor* out) { + const int ndims = in.dims(); + if (ndims == 0) return Status::OK(); + TransposePermsVec perm(ndims); + std::iota(perm.begin(), perm.end(), 0); + std::swap(perm[ndims - 2], perm[ndims - 1]); + return DoTransposeImpl(device, in, perm, conjugate, out); +} #ifdef TENSORFLOW_USE_SYCL // For SYCL lets always go through Eigen diff --git a/tensorflow/core/kernels/transpose_functor_cpu.cc b/tensorflow/core/kernels/transpose_functor_cpu.cc index b2de012be1..41b73fdaf4 100644 --- a/tensorflow/core/kernels/transpose_functor_cpu.cc +++ b/tensorflow/core/kernels/transpose_functor_cpu.cc @@ -29,17 +29,18 @@ limitations under the License. typedef Eigen::ThreadPoolDevice CPUDevice; namespace tensorflow { -namespace internal { +namespace { -template -void TransposeSimple(const Device& device, const Tensor& in, +template +void TransposeSimple(const CPUDevice& device, const Tensor& in, const gtl::ArraySlice perm, Tensor* out) { const int ndims = in.dims(); gtl::InlinedVector in_strides = ComputeStride(in.shape()); gtl::InlinedVector out_strides = ComputeStride(out->shape()); const T* p = reinterpret_cast(in.tensor_data().data()); T* q = reinterpret_cast(const_cast((out->tensor_data().data()))); - auto transpose_fn = [=](int64 begin, int64 end) { + auto transpose_fn = [=, &in_strides, &out_strides, &perm](int64 begin, + int64 end) { for (int64 o_idx = begin; o_idx < end; ++o_idx) { int64 i_idx = 0; int64 t = o_idx; @@ -64,7 +65,7 @@ void TransposeSimple(const Device& device, const Tensor& in, device.parallelFor(in.NumElements(), cost, std::move(transpose_fn)); } -} // end namespace internal +} // namespace template struct Transpose { @@ -88,32 +89,47 @@ struct Transpose { out); break; default: - internal::TransposeSimple(d, in, perm, out); + TransposeSimple(d, in, perm, out); break; } } }; -template <> -Status DoTranspose(const CPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - false /* conjugate */, out); -} +#define INSTANTIATE(DEVICE) \ + template <> \ + Status DoTranspose(const DEVICE& device, const Tensor& in, \ + const gtl::ArraySlice perm, Tensor* out) { \ + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/false, \ + out); \ + } \ + template <> \ + Status DoConjugateTranspose(const DEVICE& device, const Tensor& in, \ + const gtl::ArraySlice perm, \ + Tensor* out) { \ + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/true, \ + out); \ + } \ + template <> \ + Status DoMatrixTranspose(const DEVICE& device, const Tensor& in, \ + Tensor* out) { \ + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/false, \ + out); \ + } \ + template <> \ + Status DoConjugateMatrixTranspose(const DEVICE& device, const Tensor& in, \ + Tensor* out) { \ + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/true, \ + out); \ + } -template <> -Status DoConjugateTranspose(const CPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - true /* conjugate */, out); -} +INSTANTIATE(CPUDevice) #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; namespace internal { -template -void TransposeSYCL(const Device& d, const Tensor& in, +template +void TransposeSYCL(const SYCLDevice& d, const Tensor& in, const gtl::ArraySlice perm, bool conjugate, Tensor* out) { switch (in.dims()) { @@ -165,19 +181,11 @@ struct Transpose { } }; -template <> -Status DoTranspose(const SYCLDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - false /* conjugate */, out); -} +// Explicit instantiation. +template struct Transpose; -template <> -Status DoConjugateTranspose(const SYCLDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - true /* conjugate */, out); -} +INSTANTIATE(SYCLDevice) +#undef INSTANTIATE #endif // TENSORFLOW_USE_SYCL diff --git a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc index 364baf9a51..493dac9a7c 100644 --- a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc +++ b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc @@ -53,8 +53,8 @@ __global__ void TransposeKernel(int nthreads, const T* src, const int32* buf, } } -template -void TransposeSimple(const Device& d, const Tensor& in, +template +void TransposeSimple(const GPUDevice& d, const Tensor& in, const gtl::ArraySlice perm, Tensor* out) { // Ensures we can use 32-bit index. const int64 nelem = in.NumElements(); @@ -165,23 +165,9 @@ struct TransposeUsingTile { } }; -} // end namespace internal - -template <> -Status DoTranspose(const GPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - false /* conjugate */, out); -} - -template <> -Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - true /* conjugate */, out); -} +} // namespace internal -// Transpose kernel specialized for CPU Device. +// Transpose kernel specialized for GPU Device. template struct Transpose { static void run(const GPUDevice& d, const Tensor& in, @@ -216,19 +202,43 @@ struct Transpose { } break; default: - internal::TransposeSimple(d, in, perm, out); + internal::TransposeSimple(d, in, perm, out); break; } } }; -template <> -struct Transpose { +template +struct Transpose { static void run(const GPUDevice& d, const Tensor& in, const gtl::ArraySlice perm, Tensor* out) { LOG(FATAL) << "Transpose of DT_STRING tensor not supported on GPU."; } }; +// Explicit instantiation. +template struct Transpose; + +template <> +Status DoTranspose(const GPUDevice& device, const Tensor& in, + const gtl::ArraySlice perm, Tensor* out) { + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/false, out); +} +template <> +Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in, + const gtl::ArraySlice perm, Tensor* out) { + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/true, out); +} +template <> +Status DoMatrixTranspose(const GPUDevice& device, const Tensor& in, + Tensor* out) { + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/false, out); +} +template <> +Status DoConjugateMatrixTranspose(const GPUDevice& device, const Tensor& in, + Tensor* out) { + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/true, out); +} + } // namespace tensorflow #endif // GOOGLE_CUDA -- GitLab From 34a4b21f8f9dea64d3e99a97f639396f2d5556d3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 16:19:30 -0700 Subject: [PATCH 083/573] Change GBDTClassifer to internally use twice differntiable implementation of multiclass cross entropy loss. PiperOrigin-RevId: 172532288 --- .../estimator_batch/estimator.py | 15 +++++++- .../contrib/boosted_trees/examples/mnist.py | 34 ++----------------- .../boosted_trees/python/utils/losses.py | 5 ++- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py index f8028acbdb..01752416b3 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py @@ -19,8 +19,10 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.boosted_trees.estimator_batch import model +from tensorflow.contrib.boosted_trees.python.utils import losses from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib +from tensorflow.python.ops import math_ops class GradientBoostedDecisionTreeClassifier(estimator.Estimator): @@ -65,10 +67,21 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): Raises: ValueError: If learner_config is not valid. """ + if n_classes > 2: + # For multi-class classification, use our loss implementation that + # supports second order derivative. + def loss_fn(labels, logits, weights=None): + result = losses.per_example_maxent_loss( + labels=labels, logits=logits, weights=weights, + num_classes=n_classes) + return math_ops.reduce_mean(result[0]) + else: + loss_fn = None head = head_lib.multi_class_head( n_classes=n_classes, weight_column_name=weight_column_name, - enable_centered_bias=False) + enable_centered_bias=False, + loss_fn=loss_fn) if learner_config.num_classes == 0: learner_config.num_classes = n_classes elif learner_config.num_classes != n_classes: diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py index a3b1cb5154..0539d77720 100644 --- a/tensorflow/contrib/boosted_trees/examples/mnist.py +++ b/tensorflow/contrib/boosted_trees/examples/mnist.py @@ -35,18 +35,13 @@ from __future__ import division from __future__ import print_function import argparse -import functools import sys import numpy as np import tensorflow as tf -from tensorflow.contrib import metrics as metrics_lib -from tensorflow.contrib.boosted_trees.estimator_batch import custom_loss_head -from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeEstimator +from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.python.utils import losses from tensorflow.contrib.learn import learn_runner -from tensorflow.python.ops import math_ops def get_input_fn(dataset_split, @@ -88,36 +83,13 @@ def _get_tfbt(output_dir): learner_config.growing_mode = growing_mode run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300) - # Use Cross Entropy loss (the impl in losses is twice differentiable). - loss_fn = functools.partial( - losses.per_example_maxent_loss, num_classes=num_classes) - logit_dim = num_classes learner_config.multi_class_strategy = ( learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - # Since we use custom head, we need to tell how accuracy is calculated. - def _multiclass_metrics(predictions, labels, weights): - """Prepares eval metrics for multiclass eval.""" - metrics = dict() - logits = predictions["scores"] - classes = math_ops.argmax(logits, 1) - metrics["accuracy"] = metrics_lib.streaming_accuracy( - classes, labels, weights) - return metrics - - metrics_fn = _multiclass_metrics - # Use custom loss head so we can provide our loss (cross entropy for - # multiclass). - head = custom_loss_head.CustomLossHead( - loss_fn=loss_fn, - link_fn=tf.identity, - logit_dimension=logit_dim, - metrics_fn=metrics_fn) - # Create a TF Boosted trees estimator that can take in custom loss. - estimator = GradientBoostedDecisionTreeEstimator( + estimator = GradientBoostedDecisionTreeClassifier( learner_config=learner_config, - head=head, + n_classes=num_classes, examples_per_layer=FLAGS.examples_per_layer, model_dir=output_dir, num_trees=FLAGS.num_trees, diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py index 4f128b2301..1e8b3ac08a 100644 --- a/tensorflow/contrib/boosted_trees/python/utils/losses.py +++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py @@ -101,7 +101,10 @@ def per_example_maxent_loss(labels, weights, logits, num_classes, eps=1e-15): unweighted_loss = array_ops.expand_dims(-math_ops.log(probs_for_real_class), 1) - return unweighted_loss * weights, control_flow_ops.no_op() + if weights is None: + return unweighted_loss, control_flow_ops.no_op() + else: + return unweighted_loss * weights, control_flow_ops.no_op() def per_example_squared_loss(labels, weights, predictions): -- GitLab From 0bf77c23c04394c25380fd09027d729388ebfba4 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Tue, 17 Oct 2017 16:28:53 -0700 Subject: [PATCH 084/573] Pass context handle, device name to EagerTensor constructor in convert_to_mixed_eager_tensors PiperOrigin-RevId: 172533449 --- tensorflow/python/eager/BUILD | 5 +++-- tensorflow/python/eager/core_test.py | 12 ++++++++++++ tensorflow/python/eager/execute.py | 7 +++++-- tensorflow/python/kernel_tests/BUILD | 1 + tensorflow/python/kernel_tests/logging_ops_test.py | 2 ++ 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 5a2592287c..9e9a7f4c59 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -93,7 +93,6 @@ cuda_py_test( ":test", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", - "//third_party/py/numpy", ], ) @@ -123,10 +122,12 @@ cuda_py_test( ":core", ":execute", ":test", + "//third_party/py/numpy", "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:pywrap_tensorflow", ], ) diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py index 041d388fad..1de72240e3 100644 --- a/tensorflow/python/eager/core_test.py +++ b/tensorflow/python/eager/core_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import threading +import numpy as np + from tensorflow.core.protobuf import config_pb2 from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import context @@ -29,6 +31,7 @@ from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -460,6 +463,15 @@ class TFETest(test_util.TensorFlowTestCase): with context.device('pu:0'): _ = constant_op.constant(1) + def testConvertMixedEagerTensors(self): + array = np.zeros((), dtype=np.float32) + tensor = constant_op.constant(0., dtype=dtypes.float32) + types, tensors = execute_lib.convert_to_mixed_eager_tensors( + [array, tensor], context.context()) + for typ, t in zip(types, tensors): + self.assertEquals(typ, dtypes.float32) + self.assertIsInstance(t, ops.EagerTensor) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py index 04634daba4..983c1ea73e 100644 --- a/tensorflow/python/eager/execute.py +++ b/tensorflow/python/eager/execute.py @@ -198,8 +198,11 @@ def args_to_matching_eager(l, ctx, default_dtype=None): def convert_to_mixed_eager_tensors(values, ctx): - v = [t if isinstance(t, ops.EagerTensor) else ops.EagerTensor(t, ctx) - for t in values] + v = [ + t if isinstance(t, ops.EagerTensor) else ops.EagerTensor( + t, context=ctx._handle, device=ctx.device_name) # pylint: disable=protected-access + for t in values + ] types = [t.dtype for t in v] return types, v diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index f6ecd1f0b8..847c078971 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -462,6 +462,7 @@ tf_py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py index 7fe65c57cc..28c85fa13a 100644 --- a/tensorflow/python/kernel_tests/logging_ops_test.py +++ b/tensorflow/python/kernel_tests/logging_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import logging_ops @@ -58,6 +59,7 @@ class LoggingOpsTest(test.TestCase): class PrintGradientTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testPrintShape(self): inp = constant_op.constant(2.0, shape=[100, 32]) inp_printed = logging_ops.Print(inp, [inp]) -- GitLab From 33ce1d06393f773c5317bb38ab996c2a7b8aa429 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Tue, 17 Oct 2017 16:58:02 -0700 Subject: [PATCH 085/573] Remove unused variable `i_` batch dataset ops. PiperOrigin-RevId: 172536770 --- tensorflow/core/kernels/batch_dataset_op.cc | 1 - tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc | 1 - tensorflow/core/kernels/padded_batch_dataset_op.cc | 1 - 3 files changed, 3 deletions(-) diff --git a/tensorflow/core/kernels/batch_dataset_op.cc b/tensorflow/core/kernels/batch_dataset_op.cc index 631840081f..04a41451ea 100644 --- a/tensorflow/core/kernels/batch_dataset_op.cc +++ b/tensorflow/core/kernels/batch_dataset_op.cc @@ -181,7 +181,6 @@ class BatchDatasetOp : public UnaryDatasetOpKernel { private: mutex mu_; - int64 i_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; diff --git a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc index 0174c8dfc8..e80d11eaea 100644 --- a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc +++ b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc @@ -245,7 +245,6 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel { private: mutex mu_; - int64 i_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; diff --git a/tensorflow/core/kernels/padded_batch_dataset_op.cc b/tensorflow/core/kernels/padded_batch_dataset_op.cc index 7737f57b68..cfc77690b5 100644 --- a/tensorflow/core/kernels/padded_batch_dataset_op.cc +++ b/tensorflow/core/kernels/padded_batch_dataset_op.cc @@ -349,7 +349,6 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel { private: mutex mu_; - int64 i_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; -- GitLab From 5f865f703621fed07925b3828f4a731066d98fd6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 17:47:40 -0700 Subject: [PATCH 086/573] The new class will be used as the base class for the existing 2-4 dimensional array classes to share code as well as for creating higher dimensional arrays. The API of the new class is kept compatible with the previous API to limit the scope of this change. PiperOrigin-RevId: 172543319 --- tensorflow/compiler/xla/BUILD | 23 ++ tensorflow/compiler/xla/array.h | 324 ++++++++++++++++++++++++++ tensorflow/compiler/xla/array2d.h | 131 ++--------- tensorflow/compiler/xla/array3d.h | 94 +------- tensorflow/compiler/xla/array4d.h | 185 ++------------- tensorflow/compiler/xla/array_test.cc | 145 ++++++++++++ 6 files changed, 528 insertions(+), 374 deletions(-) create mode 100644 tensorflow/compiler/xla/array.h create mode 100644 tensorflow/compiler/xla/array_test.cc diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 6c4c970ce8..be87506d3c 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -335,12 +335,33 @@ cc_library( ], ) +cc_library( + name = "array", + hdrs = ["array.h"], + deps = [ + ":types", + ":util", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "array_test", + srcs = ["array_test.cc"], + deps = [ + ":array", + ":test", + "//tensorflow/core:test_main", + ], +) + cc_library( name = "array2d", srcs = ["array2d.cc"], hdrs = ["array2d.h"], visibility = ["//visibility:public"], deps = [ + ":array", ":types", ":util", "//tensorflow/core:lib", @@ -362,6 +383,7 @@ cc_library( hdrs = ["array3d.h"], visibility = [":friends"], deps = [ + ":array", ":types", "//tensorflow/core:lib", ], @@ -383,6 +405,7 @@ cc_library( hdrs = ["array4d.h"], visibility = [":friends"], deps = [ + ":array", ":array2d", ":types", "//tensorflow/core:lib", diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h new file mode 100644 index 0000000000..3be7060a83 --- /dev/null +++ b/tensorflow/compiler/xla/array.h @@ -0,0 +1,324 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_ARRAY_H_ +#define TENSORFLOW_COMPILER_XLA_ARRAY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/core/bits.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +// General N dimensional array class with arbitrary value type. +template +class Array { + public: + // Creates a new array with the specified dimensions. + explicit Array(const std::vector& sizes) : Array(sizes, T()) {} + + // Creates a new array with the specified dimensions and specified value for + // every cell. + Array(const std::vector& sizes, T value) + : sizes_(sizes), values_(new T[num_elements()]) { + Fill(value); + } + + // Creates a 2D array from the given nested initializer list. The outer + // initializer list is the first dimension, the inner is the second dimension. + // For example, {{1, 2, 3}, {4, 5, 6}} results in an array with n1=2 and n2=3. + Array(std::initializer_list> values) + : Array(ToInt64Vector({values.size(), values.begin()->size()})) { + int64 idx = 0; + for (const auto& it1 : values) { + for (const auto& it2 : it1) { + values_[idx] = it2; + ++idx; + } + } + CHECK(idx == num_elements()); + } + + // Creates a 3D array from the given nested initializer list. The outer + // initializer list is the first dimension, and so on. + Array(std::initializer_list>> + values) + : Array(ToInt64Vector({values.size(), values.begin()->size(), + values.begin()->begin()->size()})) { + int64 idx = 0; + for (const auto& it1 : values) { + for (const auto& it2 : it1) { + for (const auto& it3 : it2) { + values_[idx] = it3; + ++idx; + } + } + } + CHECK(idx == num_elements()); + } + + // Creates a 4D array from the given nested initializer list. The outer + // initializer list is the first dimension, and so on. + Array(std::initializer_list< + std::initializer_list>>> + values) + : Array(ToInt64Vector({values.size(), values.begin()->size(), + values.begin()->begin()->size(), + values.begin()->begin()->begin()->size()})) { + int64 idx = 0; + for (const auto& it1 : values) { + for (const auto& it2 : it1) { + for (const auto& it3 : it2) { + for (const auto& it4 : it3) { + values_[idx] = it4; + ++idx; + } + } + } + } + CHECK(idx == num_elements()); + } + + Array(const Array& other) + : sizes_(other.sizes_), values_(new T[num_elements()]) { + std::copy(&other.values_[0], &other.values_[0] + num_elements(), + &values_[0]); + } + + Array& operator=(const Array& other) { + sizes_ = other.sizes_; + values_.reset(new T[num_elements()]); + std::copy(&other.values_[0], &other.values_[0] + num_elements(), + &values_[0]); + return *this; + } + + // Fills the array with the specified value. + void Fill(const T& value) { + std::fill(&values_[0], &values_[0] + num_elements(), value); + } + + // Fills the array with sequentially increasing values. + void FillIota(const T& value) { + std::iota(&values_[0], &values_[0] + num_elements(), value); + } + + // Fills the array with the sequence i*multiplier for i=0,1,... + void FillWithMultiples(const T& multiplier) { + for (int64 i = 0; i < num_elements(); ++i) { + values_[i] = i * multiplier; + } + } + + // Fills the array with random normal variables with the specified mean. + void FillRandom(const T& value, const double mean = 0.0, + const int seed = 12345) { + std::mt19937 g(seed); + std::normal_distribution distribution(mean, + static_cast(value)); + for (int64 i = 0; i < num_elements(); ++i) { + values_[i] = static_cast(distribution(g)); + } + } + + // Sets all the values in the array to values specified in the container. + template > + void SetValues(const Container& container) { + CHECK_EQ(std::distance(std::begin(container), std::end(container)), + num_elements()); + std::copy(std::begin(container), std::end(container), &values_[0]); + } + + // Invokes a callback with the (indices, value_ptr) for each cell in the + // array. + void Each(std::function, T*)> f) { + std::vector index(sizes_.size()); + for (int64 i = 0; i < num_elements(); ++i, next_index(&index)) { + f(index, &values_[i]); + } + } + + // Invokes a callback with the (indices, value) for each cell in the array. + void Each( + std::function, T)> f) const { + std::vector index(sizes_.size()); + for (int64 i = 0; i < num_elements(); ++i, next_index(&index)) { + f(index, values_[i]); + } + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + template + const T& operator()(Dims... dims) const { + // We are using a std::array to avoid having to allocate memory in this + // function for performance reasons. + std::array indexes{{static_cast(dims)...}}; + return values_[calculate_index(indexes)]; + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + template + T& operator()(Dims... dims) { + // We are using a std::array to avoid having to allocate memory in this + // function for performance reasons. + std::array indexes{{static_cast(dims)...}}; + return values_[calculate_index(indexes)]; + } + + // Low-level accessor for stuff like memcmp, handle with care. Returns pointer + // to the underlying storage of the array (similarly to std::vector::data()). + T* data() const { + // TODO(tberghammer): Get rid of the const_cast. Currently it is needed + // because the Eigen backend needs a non-const pointers even for reading + // from the array. + return const_cast(this)->values_.get(); + } + + // Returns the size of the dimension at the given index. + int64 dim(int64 n) const { + CHECK(n < sizes_.size()); + return sizes_[n]; + } + + // Returns a vector containing the dimensions of the array. + const std::vector& dimensions() const { return sizes_; } + + int64 num_dimensions() const { return sizes_.size(); } + + // Returns the total number of elements in the array. + int64 num_elements() const { + return std::accumulate(sizes_.begin(), sizes_.end(), 1, + std::multiplies()); + } + + bool operator==(const Array& other) const { + if (sizes_.size() != other.sizes_.size()) { + return false; + } + for (int64 i = 0; i < sizes_.size(); ++i) { + if (sizes_[i] != other.sizes_[i]) { + return false; + } + } + for (int64 i = 0; i < num_elements(); ++i) { + if (values_[i] != other.values_[i]) { + return false; + } + } + return true; + } + + bool operator!=(const Array& other) const { return !(*this == other); } + + // Returns a string representation of the array suitable for debugging. + string ToString() const { + std::vector pieces; + std::vector index(sizes_.size()); + do { + // Emit leading spaces and opening square brackets + if (index.back() == 0) { + for (int64 i = sizes_.size() - 1; i >= 0; --i) { + if (i == 0 || index[i - 1] != 0) { + for (int64 j = 0; j < sizes_.size(); ++j) { + pieces.push_back(j < i ? " " : "["); + } + break; + } + } + } + + pieces.push_back( + tensorflow::strings::AlphaNum(values_[calculate_index(index)]) + .data()); + + // Emit comma if it isn't the last element + if (index.back() != sizes_.back() - 1) { + pieces.push_back(", "); + } + + // Emit closing square brackets + for (int64 i = sizes_.size() - 1; i >= 0; --i) { + if (index[i] != sizes_[i] - 1) { + break; + } + pieces.push_back("]"); + if (i != 0 && index[i - 1] != sizes_[i - 1] - 1) { + pieces.push_back(",\n"); + } + } + } while (next_index(&index)); + return tensorflow::str_util::Join(pieces, ""); + } + + private: + // Converts an initializer_list of type U to a vector of type int64. Used by + // the initializer list based constructors to convert the size type into int64 + // to be passed to the size based constructor. + template + static std::vector ToInt64Vector( + const std::initializer_list& data) { + return std::vector(data.begin(), data.end()); + } + + // Returns the linear index from the list of per-dimension indexes. Function + // is templated so can be used with an std::array from operator() to avoid + // memory allocation. + template + int64 calculate_index(const U& indexes) const { + CHECK_EQ(sizes_.size(), indexes.size()); + int64 index = 0; + for (int64 i = 0; i < sizes_.size(); ++i) { + index *= sizes_[i]; + index += indexes[i]; + } + return index; + } + + // Advances the specified set of indexes and returns true if we haven't + // wrapped around (i.e. result isnt {0, 0, ...}). + bool next_index(std::vector* index) const { + CHECK_EQ(index->size(), sizes_.size()); + for (int64 i = sizes_.size() - 1; i >= 0; --i) { + (*index)[i]++; + if ((*index)[i] < sizes_[i]) { + return true; + } + (*index)[i] = 0; + } + return false; + } + + std::vector sizes_; + std::unique_ptr values_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_ARRAY_H_ diff --git a/tensorflow/compiler/xla/array2d.h b/tensorflow/compiler/xla/array2d.h index 2737764cbd..bb85fbee9b 100644 --- a/tensorflow/compiler/xla/array2d.h +++ b/tensorflow/compiler/xla/array2d.h @@ -24,6 +24,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/lib/core/bits.h" #include "tensorflow/core/lib/strings/str_util.h" @@ -34,93 +35,30 @@ limitations under the License. namespace xla { -// Simple 2D array structure. -// -// The data layout in major-to-minor order is: n1, n2. template -class Array2D { +class Array2D : public Array { public: - // Creates an empty array. - Array2D() : n1_(0), n2_(0) {} + Array2D() : Array(std::vector{0, 0}) {} - // Creates an array of dimensions n1 x n2, uninitialized values. Array2D(const int64 n1, const int64 n2) - : n1_(n1), n2_(n2), values_(new T[n1 * n2]()) { - Fill(T()); - } + : Array(std::vector{n1, n2}) {} - // Creates an array of dimensions n1 x n2, initialized to value. Array2D(const int64 n1, const int64 n2, const T value) - : n1_(n1), n2_(n2), values_(new T[n1 * n2]()) { - Fill(value); - } + : Array({n1, n2}, value) {} // Creates an array from the given nested initializer list. The outer // initializer list is the first dimension; the inner is the second dimension. // For example, {{1, 2, 3}, {4, 5, 6}} results in an array with n1=2 and n2=3. Array2D(std::initializer_list> values) - : Array2D(values.size(), values.begin()->size()) { - int64 n1 = 0; - for (auto n1_it = values.begin(); n1_it != values.end(); ++n1_it, ++n1) { - int64 n2 = 0; - for (auto n2_it = n1_it->begin(); n2_it != n1_it->end(); ++n2_it, ++n2) { - (*this)(n1, n2) = *n2_it; - } - } - } + : Array(values) {} - Array2D(const Array2D& other) : Array2D(other.n1(), other.n2()) { - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - } - - Array2D& operator=(const Array2D& other) { - n1_ = other.n1(); - n2_ = other.n2(); - values_.reset(new T[num_elements()]); - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - return *this; - } + Array2D(const Array2D& other) : Array(other) {} - T& operator()(const int64 i1, const int64 i2) { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - return values_[i1 * n2_ + i2]; - } + int64 n1() const { return this->dim(0); } + int64 n2() const { return this->dim(1); } - const T& operator()(const int64 i1, const int64 i2) const { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - return values_[i1 * n2_ + i2]; - } - - // Access to the array's dimensions. height() and width() provide the - // canonical interpretation of the array n1 x n2 having n1 rows of n2 columns - // each (height is number of rows; width is number of columns). - int64 n1() const { return n1_; } - int64 n2() const { return n2_; } - int64 height() const { return n1_; } - int64 width() const { return n2_; } - int64 num_elements() const { return n1_ * n2_; } - - // Low-level accessor for stuff like memcmp, handle with care. Returns pointer - // to the underlying storage of the array (similarly to std::vector::data()). - T* data() const { return const_cast(this)->values_.get(); } - - // Fills the array with the given value. - void Fill(const T& value) { - std::fill(&values_[0], &values_[0] + num_elements(), value); - } - - // Applies f to all cells in this array, in row-major order. - void Each(std::function f) { - for (int64 i0 = 0; i0 < n1(); ++i0) { - for (int64 i1 = 0; i1 < n2(); ++i1) { - f(i0, i1, &(*this)(i0, i1)); - } - } - } + int64 height() const { return this->dim(0); } + int64 width() const { return this->dim(1); } // Fills the array with a pattern of values of the form: // @@ -136,55 +74,14 @@ class Array2D { } } - // Fills the array with random normal variables of deviation value. - void FillRandom(const T& value, const double mean = 0.0, - const int seed = 12345) { - std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(value)); - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); - } - } - - // Returns a readable string representation of the array. - string ToString() const { - std::vector pieces = {"["}; - for (int64 row = 0; row < height(); ++row) { - pieces.push_back("["); - for (int64 col = 0; col < width(); ++col) { - pieces.push_back(tensorflow::strings::StrCat((*this)(row, col))); - pieces.push_back(", "); - } - pieces.pop_back(); - pieces.push_back("]"); - pieces.push_back(",\n "); - } - pieces.pop_back(); - pieces.push_back("]"); - return tensorflow::str_util::Join(pieces, ""); - } - - bool operator==(const Array2D& other) const { - if (n1() != other.n1() || n2() != other.n2()) { - return false; - } + // Applies f to all cells in this array, in row-major order. + void Each(std::function f) { for (int64 i0 = 0; i0 < n1(); ++i0) { for (int64 i1 = 0; i1 < n2(); ++i1) { - if ((*this)(i0, i1) != other(i0, i1)) { - return false; - } + f(i0, i1, &(*this)(i0, i1)); } } - return true; } - - bool operator!=(const Array2D& other) const { return !(*this == other); } - - private: - int64 n1_; - int64 n2_; - std::unique_ptr values_; }; // Returns a linspace-populated Array2D in the range [from, to] (inclusive) diff --git a/tensorflow/compiler/xla/array3d.h b/tensorflow/compiler/xla/array3d.h index 124ccd1975..e9449f01ad 100644 --- a/tensorflow/compiler/xla/array3d.h +++ b/tensorflow/compiler/xla/array3d.h @@ -24,6 +24,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -32,22 +33,16 @@ limitations under the License. namespace xla { // Simple 3D array structure. -// -// The data layout in major-to-minor order is: n1, n2, n3. template -class Array3D { +class Array3D : public Array { public: // Creates an array of dimensions n1 x n2 x n3, uninitialized values. Array3D(const int64 n1, const int64 n2, const int64 n3) - : n1_(n1), n2_(n2), n3_(n3), values_(new T[n1 * n2 * n3]) { - Fill(T()); - } + : Array(std::vector{n1, n2, n3}) {} // Creates an array of dimensions n1 x n2 x n3, initialized to value. Array3D(const int64 n1, const int64 n2, const int64 n3, const T value) - : n1_(n1), n2_(n2), n3_(n3), values_(new T[n1 * n2 * n3]) { - Fill(value); - } + : Array(std::vector{n1, n2, n3}, value) {} // Creates an array from the given nested initializer list. The outer // initializer list is the first dimension, and so on. @@ -58,84 +53,11 @@ class Array3D { // results in an array with n1=3, n2=4, n3=2. Array3D(std::initializer_list>> values) - : Array3D(values.size(), values.begin()->size(), - values.begin()->begin()->size()) { - int64 n1 = 0; - for (auto n1_it = values.begin(); n1_it != values.end(); ++n1_it, ++n1) { - int64 n2 = 0; - for (auto n2_it = n1_it->begin(); n2_it != n1_it->end(); ++n2_it, ++n2) { - int64 n3 = 0; - for (auto n3_it = n2_it->begin(); n3_it != n2_it->end(); - ++n3_it, ++n3) { - (*this)(n1, n2, n3) = *n3_it; - } - } - } - } + : Array(values) {} - Array3D(const Array3D& other) - : Array3D(other.n1(), other.n2(), other.n3()) { - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - } - - Array3D& operator=(const Array3D& other) { - n1_ = other.n1(); - n2_ = other.n2(); - n3_ = other.n3(); - values_.reset(new T[num_elements()]); - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - return *this; - } - - T& operator()(const int64 i1, const int64 i2, const int64 i3) { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - CHECK_LT(i3, n3_); - return values_[i1 * n2_ * n3_ + i2 * n3_ + i3]; - } - - const T& operator()(const int64 i1, const int64 i2, const int64 i3) const { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - CHECK_LT(i3, n3_); - return values_[i1 * n2_ * n3_ + i2 * n3_ + i3]; - } - - // Access to the array's dimensions. - int64 n1() const { return n1_; } - int64 n2() const { return n2_; } - int64 n3() const { return n3_; } - int64 num_elements() const { return n1_ * n2_ * n3_; } - - // Fills the array with the given value. - void Fill(const T& value) { - std::fill(&values_[0], &values_[0] + num_elements(), value); - } - - // Fills the array with sequentially increasing values. - void FillIota(const T& value) { - std::iota(&values_[0], &values_[0] + num_elements(), value); - } - - // Fills the array with random normal values with a mean of 0 and standard - // deviation of value. - void FillRandom(const T& value, const double mean = 0.0, - const int seed = 12345) { - std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(value)); - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); - } - } - - private: - int64 n1_; - int64 n2_; - int64 n3_; - std::unique_ptr values_; + int64 n1() const { return this->dim(0); } + int64 n2() const { return this->dim(1); } + int64 n3() const { return this->dim(2); } }; } // namespace xla diff --git a/tensorflow/compiler/xla/array4d.h b/tensorflow/compiler/xla/array4d.h index 4c7fce1aaf..f8b2b2afe5 100644 --- a/tensorflow/compiler/xla/array4d.h +++ b/tensorflow/compiler/xla/array4d.h @@ -26,6 +26,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/lib/gtl/array_slice.h" @@ -53,23 +54,15 @@ namespace xla { // more than one name is given above. See operator() for the exact // calculation of 1d indices from 4d indices. template -class Array4D { +class Array4D : public Array { public: // Creates a 4D array, uninitialized values. Array4D(int64 planes, int64 depth, int64 height, int64 width) - : planes_(planes), - depth_(depth), - height_(height), - width_(width), - values_(new T[planes * depth * height * width]) { - Fill(T()); - } + : Array(std::vector{planes, depth, height, width}) {} // Creates a 4D array, initialized to value. Array4D(int64 planes, int64 depth, int64 height, int64 width, T value) - : Array4D(planes, depth, height, width) { - Fill(value); - } + : Array(std::vector{planes, depth, height, width}, value) {} // Creates a 4D array, filled with values. // @@ -80,144 +73,26 @@ class Array4D { Array4D(int64 planes, int64 depth, int64 height, int64 width, const Container& values) : Array4D(planes, depth, height, width) { - SetValues(values); + this->SetValues(values); } // Construct an Array4D with the given nested initializer list. Array4D(std::initializer_list>>> values) - : Array4D(values.size(), values.begin()->size(), - values.begin()->begin()->size(), - values.begin()->begin()->begin()->size()) { - int64 plane = 0; - for (const auto values_in_plane : values) { - DCHECK_EQ(values_in_plane.size(), depth_); - int64 depth = 0; - for (const auto values_in_depth : values_in_plane) { - DCHECK_EQ(values_in_depth.size(), height_); - int64 height = 0; - for (const auto values_in_height : values_in_depth) { - DCHECK_EQ(values_in_height.size(), width_); - int64 width = 0; - for (const auto element_value : values_in_height) { - (*this)(plane, depth, height, width) = element_value; - ++width; - } - ++height; - } - ++depth; - } - ++plane; - } - } - - Array4D(const Array4D& other) - : Array4D(other.planes(), other.depth(), other.height(), other.width()) { - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - } - - Array4D& operator=(const Array4D& other) { - planes_ = other.planes(); - depth_ = other.depth(); - height_ = other.height(); - width_ = other.width(); - values_.reset(new T[num_elements()]); - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - return *this; - } - - T& operator()(int64 plane, int64 depth, int64 height, int64 width) { - CHECK_LT(plane, planes_); - CHECK_LT(depth, depth_); - CHECK_LT(height, height_); - CHECK_LT(width, width_); - return values_[plane * (depth_ * height_ * width_) + - depth * (height_ * width_) + height * (width_) + width]; - } - const T& operator()(int64 plane, int64 depth, int64 height, - int64 width) const { - return const_cast(this)->operator()(plane, depth, height, width); - } - - int64 width() const { return width_; } - int64 height() const { return height_; } - int64 depth() const { return depth_; } - int64 planes() const { return planes_; } + : Array(values) {} // Numerically-named aliases for the various dimensions. This matches the // dimension names used in array3d. - int64 n4() const { return width_; } - int64 n3() const { return height_; } - int64 n2() const { return depth_; } - int64 n1() const { return planes_; } - int64 num_elements() const { return width_ * height_ * depth_ * planes_; } - - // Sets all the values in the array to values. - template > - void SetValues(const Container& container) { - CHECK_EQ(std::distance(std::begin(container), std::end(container)), - num_elements()); - std::copy(std::begin(container), std::end(container), &values_[0]); - } - - // Fills the array with the given value. - void Fill(const T& value) { - std::fill(&values_[0], &values_[0] + num_elements(), value); - } + int64 n4() const { return this->dim(3); } + int64 n3() const { return this->dim(2); } + int64 n2() const { return this->dim(1); } + int64 n1() const { return this->dim(0); } - // Fills the array with iota. - void FillIota(const T& value) { - std::iota(&values_[0], &values_[0] + num_elements(), value); - } - - // Fills the array with random variable with a deviation of value and a mean - // of mean. - void FillRandom(const T& value, const double mean = 0.0, - const int seed = 12345) { - std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(value)); - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); - } - } - - // Fills values with the sequence i*multiplier for i=0,1,... - void FillWithMultiples(float multiplier) { - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = i * multiplier; - } - } - - // Invokes a callback with the (indices, value_ptr) for each cell in the 4D - // array. - void Each(std::function, T*)> f) { - for (int64 plane = 0; plane < planes(); ++plane) { - for (int64 depth = 0; depth < this->depth(); ++depth) { - for (int64 height = 0; height < this->height(); ++height) { - for (int64 width = 0; width < this->width(); ++width) { - auto& value = (*this)(plane, depth, height, width); - f({plane, depth, height, width}, &value); - } - } - } - } - } - - // Invokes a callback with the (indices, value) for each cell in the 4D array. - void Each( - std::function, T)> f) const { - // We const_cast to be able to use the common non-const implementation, - // but prevent modification of the data by passing it by-value to the - // caller. - const_cast(this)->Each( - [&f](tensorflow::gtl::ArraySlice indices, T* value) { - f(indices, *value); - }); - } + int64 width() const { return this->dim(3); } + int64 height() const { return this->dim(2); } + int64 depth() const { return this->dim(1); } + int64 planes() const { return this->dim(0); } // Fills all of the {p,z} with the array provided, which specifies {y,x}. void FillWithYX(const Array2D& value) { @@ -267,38 +142,6 @@ class Array4D { } } } - - // Returns a string representation of the 4D array suitable for debugging. - string ToString() const { - std::vector pieces = { - tensorflow::strings::Printf("p=%lld,z=%lld,y=%lld,x=%lld {\n", planes(), - depth(), height(), width())}; - for (int64 plane = 0; plane < planes_; ++plane) { - pieces.push_back(" {\n"); - for (int64 depth = 0; depth < depth_; ++depth) { - pieces.push_back(" {\n"); - for (int64 height = 0; height < height_; ++height) { - pieces.push_back(" {"); - for (int64 width = 0; width < width_; ++width) { - pieces.push_back(tensorflow::strings::StrCat( - (*this)(plane, depth, height, width), ", ")); - } - pieces.push_back("},\n"); - } - pieces.push_back(" },\n"); - } - pieces.push_back(" },\n"); - } - pieces.push_back("}"); - return tensorflow::str_util::Join(pieces, ""); - } - - private: - int64 planes_; - int64 depth_; - int64 height_; - int64 width_; - std::unique_ptr values_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/array_test.cc b/tensorflow/compiler/xla/array_test.cc new file mode 100644 index 0000000000..093784f541 --- /dev/null +++ b/tensorflow/compiler/xla/array_test.cc @@ -0,0 +1,145 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/array.h" + +#include + +#include "tensorflow/compiler/xla/test.h" + +namespace xla { +namespace { + +TEST(ArrayTest, UninitializedDimsCtor) { + Array uninit({2, 3}); + EXPECT_EQ(uninit.num_dimensions(), 2); + EXPECT_EQ(uninit.dim(0), 2); + EXPECT_EQ(uninit.dim(1), 3); + EXPECT_EQ(uninit.num_elements(), 6); +} + +TEST(ArrayTest, FillCtor) { + Array fullof7({1, 2, 3}, 7); + + EXPECT_EQ(fullof7.dim(0), 1); + EXPECT_EQ(fullof7.dim(1), 2); + EXPECT_EQ(fullof7.dim(2), 3); + + for (int64 n0 = 0; n0 < fullof7.dim(0); ++n0) { + for (int64 n1 = 0; n1 < fullof7.dim(1); ++n1) { + for (int64 n2 = 0; n2 < fullof7.dim(2); ++n2) { + EXPECT_EQ(fullof7(n0, n1, n2), 7); + } + } + } +} + +TEST(ArrayTest, InitializerListCtor) { + Array arr({{1, 2, 3}, {4, 5, 6}}); + + EXPECT_EQ(arr.dim(0), 2); + EXPECT_EQ(arr.dim(1), 3); + + EXPECT_EQ(arr(0, 0), 1); + EXPECT_EQ(arr(0, 1), 2); + EXPECT_EQ(arr(0, 2), 3); + EXPECT_EQ(arr(1, 0), 4); + EXPECT_EQ(arr(1, 1), 5); + EXPECT_EQ(arr(1, 2), 6); +} + +TEST(ArrayTest, IndexingReadWrite) { + Array arr({2, 3}); + + EXPECT_EQ(arr(1, 1), 0); + EXPECT_EQ(arr(1, 2), 0); + arr(1, 1) = 51; + arr(1, 2) = 61; + EXPECT_EQ(arr(1, 1), 51); + EXPECT_EQ(arr(1, 2), 61); +} + +TEST(ArrayTest, IndexingReadWriteBool) { + Array arr{{false, true, false}, {false, true, false}}; + + EXPECT_EQ(arr(0, 1), true); + EXPECT_EQ(arr(0, 2), false); + arr(0, 1) = false; + arr(0, 2) = true; + EXPECT_EQ(arr(0, 1), false); + EXPECT_EQ(arr(0, 2), true); +} + +TEST(ArrayTest, Fill) { + Array fullof7({2, 3}, 7); + for (int64 n1 = 0; n1 < fullof7.dim(0); ++n1) { + for (int64 n2 = 0; n2 < fullof7.dim(1); ++n2) { + EXPECT_EQ(fullof7(n1, n2), 7); + } + } + + fullof7.Fill(11); + for (int64 n1 = 0; n1 < fullof7.dim(0); ++n1) { + for (int64 n2 = 0; n2 < fullof7.dim(1); ++n2) { + EXPECT_EQ(fullof7(n1, n2), 11); + } + } +} + +TEST(ArrayTest, DataPointer) { + Array arr{{1, 2, 3}, {4, 5, 6}}; + EXPECT_EQ(arr.data()[0], 1); +} + +TEST(ArrayTest, Stringification1D) { + Array arr({2}, 1); + const string expected = R"([1, 1])"; + EXPECT_EQ(expected, arr.ToString()); +} + +TEST(ArrayTest, Stringification2D) { + Array arr({2, 3}, 7); + const string expected = "[[7, 7, 7],\n [7, 7, 7]]"; + EXPECT_EQ(expected, arr.ToString()); +} + +TEST(ArrayTest, Stringification3D) { + Array arr({2, 3, 4}, 5); + const string expected = R"([[[5, 5, 5, 5], + [5, 5, 5, 5], + [5, 5, 5, 5]], + [[5, 5, 5, 5], + [5, 5, 5, 5], + [5, 5, 5, 5]]])"; + EXPECT_EQ(expected, arr.ToString()); +} + +TEST(ArrayTest, Each) { + Array arr({2, 3, 4}); + arr.FillWithMultiples(1); + + int64 each_count = 0, each_sum = 0; + arr.Each([&](tensorflow::gtl::ArraySlice idx, int cell) { + int64 lin_idx = idx[0] * 12 + idx[1] * 4 + idx[2]; + EXPECT_EQ(lin_idx, cell); + each_count++; + each_sum += cell; + }); + EXPECT_EQ(arr.num_elements(), each_count); + EXPECT_EQ(arr.num_elements() * (arr.num_elements() - 1) / 2, each_sum); +} + +} // namespace +} // namespace xla -- GitLab From 8c8bf69563a0d5b7f52d6153f09580946296d1f7 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 17 Oct 2017 18:31:51 -0700 Subject: [PATCH 087/573] should_use_result eager-safe. ResourceVariable.numpy() PiperOrigin-RevId: 172547480 --- .../python/kernel_tests/resource_variable_ops_test.py | 6 ++++++ tensorflow/python/ops/resource_variable_ops.py | 6 ++++++ tensorflow/python/util/tf_should_use.py | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 8cf8286ed1..ec9192b1a0 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -62,6 +62,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): "Expected float got int32."): _ = resource_variable_ops.read_variable_op(handle, dtype=dtypes.float32) + def testEagerInitializedValue(self): + with context.eager_mode(): + variable = resource_variable_ops.ResourceVariable(1.0, name="eager-init") + self.assertAllEqual(variable.numpy(), 1.0) + self.assertAllEqual(variable.initialized_value().numpy(), 1.0) + def testAssignVariableDtypeMismatchEager(self): with context.eager_mode(): handle = resource_variable_ops.var_handle_op( diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index cbfa141256..2c9a3ff19a 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -513,6 +513,12 @@ class ResourceVariable(variables.Variable): raise RuntimeError("Trying to eval in EAGER mode") return self._graph_element.eval(session=session) + def numpy(self): + if context.in_graph_mode(): + raise NotImplementedError( + "numpy() is only available when eager execution is enabled.") + return self.read_value().numpy() + def _set_save_slice_info(self, save_slice_info): """Sets the slice info for this `ResourceVariable`. diff --git a/tensorflow/python/util/tf_should_use.py b/tensorflow/python/util/tf_should_use.py index d9b2e6fcd7..99081cb294 100644 --- a/tensorflow/python/util/tf_should_use.py +++ b/tensorflow/python/util/tf_should_use.py @@ -44,6 +44,11 @@ def _add_should_use_warning(x, fatal_error=False): if x is None: # special corner case where x is None return x + # TODO(apassos) we don't have an easier way to check because importing context + # or ops here would create a BUILD dependency cycle. + if type(x).__name__ == 'EagerTensor': + return x + def override_method(method): def fn(self, *args, **kwargs): return method(self, *args, **kwargs) -- GitLab From a308914e24613a49e7f5d2550a19802e4de1283c Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 17 Oct 2017 18:32:09 -0700 Subject: [PATCH 088/573] EagerTensor.__array__ Also rewrites most eager tests to remove now-unnecessary calls to .numpy() PiperOrigin-RevId: 172547504 --- tensorflow/python/eager/backprop_test.py | 54 ++++---- tensorflow/python/eager/core_test.py | 28 ++-- tensorflow/python/eager/function_test.py | 36 ++--- .../python/eager/graph_callable_test.py | 20 +-- tensorflow/python/eager/ops_test.py | 124 +++++++++--------- tensorflow/python/eager/tape_test.py | 22 ++-- tensorflow/python/eager/tensor_test.py | 14 +- tensorflow/python/framework/ops.py | 4 + 8 files changed, 153 insertions(+), 149 deletions(-) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 2645d542c0..002be95d0f 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -90,8 +90,8 @@ class BackpropTest(test.TestCase): return math_ops.add(c, constant_op.constant(3.0)) grads_and_vars = backprop.implicit_grad(fn)() - self.assertEqual(grads_and_vars[0][0].numpy(), 1.0) - self.assertEqual(id(grads_and_vars[0][1]), id(x)) + self.assertAllEqual(grads_and_vars[0][0], 1.0) + self.assertAllEqual(id(grads_and_vars[0][1]), id(x)) def testDy(self): @@ -99,7 +99,7 @@ class BackpropTest(test.TestCase): return x grad_fn = backprop.gradients_function(f) - self.assertAllEqual(2., grad_fn(1., dy=2.)[0].numpy()) + self.assertAllEqual(2., grad_fn(1., dy=2.)[0]) def testImplicitGradOverEmbeddingLookup(self): batch_size = 8 @@ -131,13 +131,13 @@ class BackpropTest(test.TestCase): tf_opt = training.GradientDescentOptimizer(0.1) tf_embedding.initializer.run() - self.assertAllClose(tf_grad.indices.eval(), grad.indices.numpy()) - self.assertAllClose(tf_grad.values.eval(), grad.values.numpy()) + self.assertAllClose(tf_grad.indices.eval(), grad.indices) + self.assertAllClose(tf_grad.values.eval(), grad.values) tf_opt.apply_gradients([(tf_grad, tf_embedding)]).run() expected = tf_embedding.eval() opt.apply_gradients([(grad, embedding)]) - self.assertAllClose(expected, embedding.read_value().numpy()) + self.assertAllClose(expected, embedding.read_value()) def testGradientNone(self): @@ -167,7 +167,7 @@ class BackpropTest(test.TestCase): f = constant_op.constant([[0.1]]) grad = backprop.gradients_function(second, [0])(f)[0] - self.assertAllEqual([[0.0]], grad.numpy()) + self.assertAllEqual([[0.0]], grad) def testMakeVJP(self): @@ -176,8 +176,8 @@ class BackpropTest(test.TestCase): wrapped_fn = backprop.make_vjp(f) result, vjp = wrapped_fn(constant_op.constant(3.0)) - self.assertEqual(result.numpy(), 9.0) - self.assertEqual(vjp(2.0)[0].numpy(), 12.0) + self.assertAllEqual(result, 9.0) + self.assertAllEqual(vjp(2.0)[0], 12.0) def testGradGrad(self): @@ -190,7 +190,7 @@ class BackpropTest(test.TestCase): gradgrad = backprop.gradients_function(grad, [0]) - self.assertAllEqual(gradgrad(constant_op.constant(3.0))[0].numpy(), 2.0) + self.assertAllEqual(gradgrad(constant_op.constant(3.0))[0], 2.0) def testGradGradExp(self): @@ -200,7 +200,7 @@ class BackpropTest(test.TestCase): gradgrad = backprop.gradients_function(grad, [0]) - self.assertAllEqual(gradgrad(constant_op.constant(0.0))[0].numpy(), 1.0) + self.assertAllEqual(gradgrad(constant_op.constant(0.0))[0], 1.0) def testGPU(self): if not context.context().num_gpus(): @@ -215,7 +215,7 @@ class BackpropTest(test.TestCase): return math_ops.add(c, constant_op.constant(3.0)).as_cpu_tensor() grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testGPUImplicitGrad(self): if not context.context().num_gpus(): @@ -240,7 +240,7 @@ class BackpropTest(test.TestCase): return math_ops.add(c, constant_op.constant(3.0)) grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testTensorCopyGPU2CPU2GPU(self): if not context.context().num_gpus(): @@ -254,7 +254,7 @@ class BackpropTest(test.TestCase): b = constant_op.constant(2.0) grad = backprop.gradients_function(f, [0])(a, b)[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testEmptyParams(self): @@ -264,8 +264,8 @@ class BackpropTest(test.TestCase): x = constant_op.constant(1.0) y = constant_op.constant(2.0) dx, dy = backprop.gradients_function(fn)(x, y) - self.assertAllEqual(dx.numpy(), y.numpy()) - self.assertAllEqual(dy.numpy(), x.numpy()) + self.assertAllEqual(dx, y.numpy()) + self.assertAllEqual(dy, x.numpy()) def testUnconnectedNone(self): v = resource_variable_ops.ResourceVariable( @@ -285,9 +285,9 @@ class BackpropTest(test.TestCase): x = 2.0 y = 3.0 val, (dx, dy) = val_and_grads_fn(x, y) - self.assertAllClose(val.numpy(), x * y) - self.assertAllEqual(dx.numpy(), y) - self.assertAllEqual(dy.numpy(), x) + self.assertAllClose(val, x * y) + self.assertAllEqual(dx, y) + self.assertAllEqual(dy, x) def testNonEmptyParamsForValueAndGradFunction(self): def fn(a, b): @@ -297,9 +297,9 @@ class BackpropTest(test.TestCase): x = 2.0 y = 3.0 val, grads = val_and_grad_fn(x, y) - self.assertAllClose(val.numpy(), x * y) + self.assertAllClose(val, x * y) self.assertEqual(1, len(grads)) - self.assertAllEqual(grads[0].numpy(), x) + self.assertAllEqual(grads[0], x) def testTensorCopyCPU2GPU2CPU(self): if not context.context().num_gpus(): @@ -317,7 +317,7 @@ class BackpropTest(test.TestCase): b = constant_op.constant(2.0) grad = backprop.gradients_function(f, [0])(a, b)[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testGetAttrType(self): typ = backprop.op_attr_type('Add', 'T') @@ -372,7 +372,7 @@ class BackpropTest(test.TestCase): return math_ops.reduce_mean(b) grad = backprop.implicit_grad(fn)()[0][0] - self.assertAllEqual([1.0], grad.numpy()) + self.assertAllEqual([1.0], grad) def testOutput(self): @@ -382,7 +382,7 @@ class BackpropTest(test.TestCase): x = constant_op.constant([0.0, 1.0, 2.0]) grad = backprop.gradients_function(multiout)(x)[0] - self.assertAllEqual([1.0, 3.0, 5.0], grad.numpy()) + self.assertAllEqual([1.0, 3.0, 5.0], grad) def testMultiValuePreservesIfNotDiffedAgainst(self): @@ -394,7 +394,7 @@ class BackpropTest(test.TestCase): s = [1, 1, 1, 1] grad = backprop.gradients_function(tfe_conv2d, params=(0,))(i, k, s)[0] - self.assertAllEqual([[[[2.0]]]], grad.numpy()) + self.assertAllEqual([[[[2.0]]]], grad) def testSameObjectForMultipleArguments(self): @@ -483,8 +483,8 @@ class BackpropTest(test.TestCase): grads_and_vars = g() self.assertEqual(1, len(grads_and_vars)) grad, var = grads_and_vars[0] - self.assertEqual(7, grad.numpy()) - self.assertEqual(x, var) + self.assertAllEqual(7, grad) + self.assertAllEqual(x, var) def testCustomGradient(self): diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py index 1de72240e3..54a0be6dd9 100644 --- a/tensorflow/python/eager/core_test.py +++ b/tensorflow/python/eager/core_test.py @@ -151,14 +151,14 @@ class TFETest(test_util.TensorFlowTestCase): cpu = constant_op.constant([[1., 2.], [3., 4.]]) c2g = cpu.as_gpu_tensor() - self.assertAllEqual(c2g.numpy(), cpu.numpy()) + self.assertAllEqual(c2g, cpu.numpy()) def testCopyFromCPUToCPU(self): ta = constant_op.constant([[1, 2], [3, 4]]) tb = ta.as_cpu_tensor() self.assertNotEqual(id(ta), id(tb)) - self.assertAllEqual(ta.numpy(), tb.numpy()) + self.assertAllEqual(ta, tb.numpy()) def testRegisterExceptionClass(self): with self.assertRaises(TypeError): @@ -174,7 +174,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[three, five], attrs=('T', three.dtype.as_datatype_enum))[0] - self.assertEqual(15, product.numpy()) + self.assertAllEqual(15, product) def testExecuteTooManyNumOutputs(self): # num_outputs provided is 50, but only one output is produced. @@ -184,7 +184,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=50, inputs=[constant_op.constant(3), constant_op.constant(5)], attrs=('T', dtypes.int32.as_datatype_enum))[0] - self.assertEqual(15, product.numpy()) + self.assertAllEqual(15, product) def testMatMulGPU(self): if not context.context().num_gpus(): @@ -197,7 +197,7 @@ class TFETest(test_util.TensorFlowTestCase): inputs=[three, five], attrs=('transpose_a', False, 'transpose_b', False, 'T', three.dtype.as_datatype_enum))[0] - self.assertEqual([[15.0]], product.numpy()) + self.assertAllEqual([[15.0]], product) def testExecuteStringAttr(self): checked_three = execute( @@ -222,7 +222,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[constant_op.constant(3.0), constant_op.constant(2.9)], attrs=('tolerance', 0.3, 'T', dtypes.float32.as_datatype_enum))[0] - self.assertTrue(almost_equal.numpy()) + self.assertTrue(almost_equal) def testExecuteFloatAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -238,7 +238,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[constant_op.constant(3), constant_op.constant(4)], attrs=('T', dtypes.int32.as_datatype_enum, 'N', 2))[0] - self.assertEqual(7, total.numpy()) + self.assertAllEqual(7, total) def testExecuteIntAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -257,7 +257,7 @@ class TFETest(test_util.TensorFlowTestCase): constant_op.constant([[5]])], attrs=('transpose_a', True, 'transpose_b', False, 'T', dtypes.int32.as_datatype_enum))[0] - self.assertEqual([[15]], product.numpy()) + self.assertAllEqual([[15]], product) def testExecuteShapeAttr(self): execute( @@ -310,7 +310,7 @@ class TFETest(test_util.TensorFlowTestCase): inputs=[constant_op.constant([3.0, 5.0, 7.0])], attrs=('T', dtypes.float32.as_datatype_enum, 'boundaries', [4.0, 6.0]))[0] - self.assertAllEqual([0, 1, 2], b.numpy()) + self.assertAllEqual([0, 1, 2], b) def testExecuteListFloatAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -335,7 +335,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[constant_op.constant([[[3.0]]])], attrs=('T', dtypes.float32.as_datatype_enum, 'squeeze_dims', [0, 2]))[0] - self.assertAllEqual([3], b.numpy()) + self.assertAllEqual([3], b) def testExecuteListIntAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -407,9 +407,9 @@ class TFETest(test_util.TensorFlowTestCase): inputs=[constant_op.constant(split_dim), constant_op.constant(value)], attrs=('num_split', 3, 'T', dtypes.int32.as_datatype_enum)) - self.assertAllEqual([[0], [3]], x1.numpy()) - self.assertAllEqual([[1], [4]], x2.numpy()) - self.assertAllEqual([[2], [5]], x3.numpy()) + self.assertAllEqual([[0], [3]], x1) + self.assertAllEqual([[1], [4]], x2) + self.assertAllEqual([[2], [5]], x3) def testExecuteBadNumOutputsArgument(self): with self.assertRaises(TypeError): @@ -442,7 +442,7 @@ class TFETest(test_util.TensorFlowTestCase): x = constant_op.constant(1) three_x = add(add(x, x), x) self.assertEquals(dtypes.int32, three_x.dtype) - self.assertEquals(3, three_x.numpy()) + self.assertAllEqual(3, three_x) def testOperationWithNoInputsRunsOnDevice(self): if not context.context().num_gpus(): diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index e27f9ebc27..e9e396b49b 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -53,7 +53,7 @@ class FunctionTest(test.TestCase): t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) out = sq(t) - self.assertAllEqual(out.numpy(), math_ops.matmul(t, t).numpy()) + self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) def testGraphModeWithGradients(self): v = resource_variable_ops.ResourceVariable(1.0) @@ -66,7 +66,7 @@ class FunctionTest(test.TestCase): return backprop.implicit_grad(inner)()[0][0] - self.assertAllEqual(step().numpy(), 2.0) + self.assertAllEqual(step(), 2.0) def testTensorConversionWithDefun(self): @@ -74,7 +74,7 @@ class FunctionTest(test.TestCase): def f(x): return math_ops.add(x, constant_op.constant(3)) - self.assertAllEqual(5, f(constant_op.constant(2)).numpy()) + self.assertAllEqual(5, f(constant_op.constant(2))) def testTensorConversionCall(self): @@ -86,7 +86,7 @@ class FunctionTest(test.TestCase): def g(x): return f(f(x)) - self.assertAllEqual(8, g(constant_op.constant(2)).numpy()) + self.assertAllEqual(8, g(constant_op.constant(2))) def testDefunCallBackprop(self): @@ -98,7 +98,7 @@ class FunctionTest(test.TestCase): def g(x): return backprop.gradients_function(f, [0])(x)[0] - self.assertAllEqual(2, g(constant_op.constant(2)).numpy()) + self.assertAllEqual(2, g(constant_op.constant(2))) def testGraphModeEagerGradError(self): with context.graph_mode(): @@ -149,7 +149,7 @@ class FunctionTest(test.TestCase): return f(x) g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0] - self.assertEqual(g.numpy(), 1.0) + self.assertAllEqual(g, 1.0) def testGradient(self): matmul = function.defun(math_ops.matmul) @@ -159,7 +159,7 @@ class FunctionTest(test.TestCase): t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) grad_t, = backprop.gradients_function(sq, [0])(t) - self.assertAllEqual(grad_t.numpy(), [[6, 6], [14, 14]]) + self.assertAllEqual(grad_t, [[6, 6], [14, 14]]) def testGradientInFunction(self): @@ -167,7 +167,7 @@ class FunctionTest(test.TestCase): def f(x): return backprop.gradients_function(lambda y: y * y, [0])(x)[0] - self.assertEqual(f(constant_op.constant(1.0)).numpy(), 2.0) + self.assertAllEqual(f(constant_op.constant(1.0)), 2.0) def testFunctionOnDevice(self): if not context.context().num_gpus(): @@ -176,7 +176,7 @@ class FunctionTest(test.TestCase): x = constant_op.constant([1.]).as_gpu_tensor() f = function.defun(math_ops.add) y = f(x, x).as_cpu_tensor() - self.assertAllEqual(y.numpy(), [2.]) + self.assertAllEqual(y, [2.]) def testFunctionHandlesInputsOnDifferentDevices(self): if not context.context().num_gpus(): @@ -187,7 +187,7 @@ class FunctionTest(test.TestCase): value = constant_op.constant([1., 2.]).as_gpu_tensor() shape = constant_op.constant([2, 1]) reshaped = reshape(value, shape).as_cpu_tensor() - self.assertAllEqual(reshaped.numpy(), [[1], [2]]) + self.assertAllEqual(reshaped, [[1], [2]]) def testFunctionHandlesInputsPlacedOnTheWrongDeviceGracefully(self): if not context.context().num_gpus(): @@ -210,7 +210,7 @@ class FunctionTest(test.TestCase): return my_function(x)[0] g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0)) - self.assertAllEqual(g[0].numpy(), 1.) + self.assertAllEqual(g[0], 1.) def testNoneOutput(self): @@ -231,7 +231,7 @@ class FunctionTest(test.TestCase): def add_one(x): return add(x, 1) - self.assertAllEqual(3, add_one(constant_op.constant(2)).numpy()) + self.assertAllEqual(3, add_one(constant_op.constant(2))) def testSequenceInputs(self): clip_by_global_norm = function.defun(clip_ops.clip_by_global_norm) @@ -258,13 +258,13 @@ class FunctionTest(test.TestCase): constant_op.constant(5) ]) self.assertEqual(len(ret), 2) - self.assertEqual(ret[0][0].numpy(), 2) - self.assertEqual(ret[0][1][0][0].numpy(), 8) - self.assertEqual(ret[0][1][0][1].numpy(), 4) + self.assertAllEqual(ret[0][0], 2) + self.assertAllEqual(ret[0][1][0][0], 8) + self.assertAllEqual(ret[0][1][0][1], 4) self.assertTrue(isinstance(ret[0][1][0], tuple)) - self.assertEqual(ret[0][1][1].numpy(), 6) - self.assertEqual(ret[0][2].numpy(), 10) - self.assertEqual(ret[1].numpy(), 15) + self.assertAllEqual(ret[0][1][1], 6) + self.assertAllEqual(ret[0][2], 10) + self.assertAllEqual(ret[1], 15) if __name__ == '__main__': diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py index e77a33981d..548e16a909 100644 --- a/tensorflow/python/eager/graph_callable_test.py +++ b/tensorflow/python/eager/graph_callable_test.py @@ -57,7 +57,7 @@ class GraphCallableTest(test.TestCase): v.assign(x) my_function(constant_op.constant(4, dtype=dtypes.float32)) - self.assertEqual(4, my_function.variables[0].read_value().numpy()) + self.assertAllEqual(4, my_function.variables[0].read_value()) def testFunctionWithoutReturnValueAndArgs(self): @@ -68,7 +68,7 @@ class GraphCallableTest(test.TestCase): v.assign(4) my_function() - self.assertEqual(4, my_function.variables[0].read_value().numpy()) + self.assertAllEqual(4, my_function.variables[0].read_value()) def testVariableAPI(self): @@ -113,7 +113,7 @@ class GraphCallableTest(test.TestCase): v.assign(v * x) return v.read_value() - self.assertEqual(my_function(constant_op.constant(2.0)).numpy(), 6.0) + self.assertAllEqual(my_function(constant_op.constant(2.0)), 6.0) def testEmptyInitializer(self): @@ -149,7 +149,7 @@ class GraphCallableTest(test.TestCase): def f(x): return math_ops.add(x, constant_op.constant(3)) - self.assertAllEqual(5, f(constant_op.constant(2)).numpy()) + self.assertAllEqual(5, f(constant_op.constant(2))) def testNestedFunction(self): @@ -165,7 +165,7 @@ class GraphCallableTest(test.TestCase): def add_one(x): return add(x, 1) - self.assertAllEqual(3, add_one(constant_op.constant(2)).numpy()) + self.assertAllEqual(3, add_one(constant_op.constant(2))) # TODO(ashankar): Make this work. # The problem is that the two graph_callables (for add_one and add_two) @@ -187,8 +187,8 @@ class GraphCallableTest(test.TestCase): return add(x, 2) two = constant_op.constant(2) - self.assertAllEqual(3, add_one(two).numpy()) - self.assertAllEqual(4, add_two(two).numpy()) + self.assertAllEqual(3, add_one(two)) + self.assertAllEqual(4, add_two(two)) def testNestedSequenceInputs(self): sd = graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32) @@ -205,11 +205,11 @@ class GraphCallableTest(test.TestCase): constant_op.constant(4.)] ret = my_op(inputs) self.assertEqual(len(ret), 2.) - self.assertEqual(ret[1].numpy(), 10.) + self.assertAllEqual(ret[1], 10.) my_op.variables[0].assign(1.) ret = my_op(inputs) - self.assertEqual(ret[1].numpy(), 11.) + self.assertAllEqual(ret[1], 11.) def testVariableShapeIsTensorShape(self): @graph_callable.graph_callable([]) @@ -243,7 +243,7 @@ class GraphCallableTest(test.TestCase): grad_fn = backprop.implicit_grad(my_function) grads_and_vars = list(zip(*grad_fn())) - self.assertEqual(6., grads_and_vars[0][0].numpy()) + self.assertAllEqual(6., grads_and_vars[0][0]) if __name__ == "__main__": diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 78423468ea..6d1a5fe264 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -41,7 +41,7 @@ class OpsTest(test_util.TensorFlowTestCase): three = constant_op.constant(3) five = constant_op.constant(5) product = three * five - self.assertEqual(15, product.numpy()) + self.assertAllEqual(15, product) def testMatMulGPU(self): if not context.context().num_gpus(): @@ -49,7 +49,7 @@ class OpsTest(test_util.TensorFlowTestCase): three = constant_op.constant([[3.]]).as_gpu_tensor() five = constant_op.constant([[5.]]).as_gpu_tensor() product = math_ops.matmul(three, five) - self.assertEqual([[15.0]], product.numpy()) + self.assertEqual([[15.0]], product) def testExecuteStringAttr(self): three = constant_op.constant(3.0) @@ -62,27 +62,27 @@ class OpsTest(test_util.TensorFlowTestCase): almost_three = constant_op.constant(2.8) almost_equal = math_ops.approximate_equal( three, almost_three, tolerance=0.3) - self.assertTrue(almost_equal.numpy()) + self.assertTrue(almost_equal) def testExecuteIntAttr(self): three = constant_op.constant(3) four = constant_op.constant(4) total = math_ops.add_n([three, four]) - self.assertEqual(7, total.numpy()) + self.assertAllEqual(7, total) def testExecuteBoolAttr(self): three = constant_op.constant([[3]]) five = constant_op.constant([[5]]) product = math_ops.matmul(three, five, transpose_a=True) - self.assertEqual([[15]], product.numpy()) + self.assertAllEqual([[15]], product) def testExecuteOneListOutput(self): split_dim = constant_op.constant(1) value = constant_op.constant([[0, 1, 2], [3, 4, 5]]) x1, x2, x3 = array_ops.split(value, 3, axis=split_dim) - self.assertAllEqual([[0], [3]], x1.numpy()) - self.assertAllEqual([[1], [4]], x2.numpy()) - self.assertAllEqual([[2], [5]], x3.numpy()) + self.assertAllEqual([[0], [3]], x1) + self.assertAllEqual([[1], [4]], x2) + self.assertAllEqual([[2], [5]], x3) def testGraphMode(self): graph = ops.Graph() @@ -97,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase): self.skipTest('No GPUs found') with context.device('/gpu:0'): r = constant_op.constant(1) + constant_op.constant(2) - self.assertEqual(r.numpy(), 3) + self.assertEqual(r, 3) def testExecuteListOutputLen1(self): split_dim = constant_op.constant(1) @@ -105,7 +105,7 @@ class OpsTest(test_util.TensorFlowTestCase): result = array_ops.split(value, 1, axis=split_dim) self.assertTrue(isinstance(result, list)) self.assertEqual(1, len(result)) - self.assertAllEqual([[0, 1, 2], [3, 4, 5]], result[0].numpy()) + self.assertAllEqual([[0, 1, 2], [3, 4, 5]], result[0]) def testExecuteListOutputLen0(self): empty = constant_op.constant([], dtype=dtypes.int32) @@ -120,8 +120,8 @@ class OpsTest(test_util.TensorFlowTestCase): out, idx = result self.assertTrue(out is result.out) self.assertTrue(idx is result.idx) - self.assertAllEqual([2, 4, 6], out.numpy()) - self.assertAllEqual([1, 3, 5], idx.numpy()) + self.assertAllEqual([2, 4, 6], out) + self.assertAllEqual([1, 3, 5], idx) def testExecuteMultipleListOutput(self): split_dim = constant_op.constant(1, dtype=dtypes.int64) @@ -138,12 +138,12 @@ class OpsTest(test_util.TensorFlowTestCase): self.assertEqual(output_indices, result.output_indices) self.assertEqual(output_values, result.output_values) self.assertEqual(output_shape, result.output_shape) - self.assertAllEqual([[0, 2], [1, 0], [1, 1]], output_indices[0].numpy()) - self.assertAllEqual([[0, 0], [0, 1]], output_indices[1].numpy()) - self.assertAllEqual([2, 7, 11], output_values[0].numpy()) - self.assertAllEqual([3, 5], output_values[1].numpy()) - self.assertAllEqual([2, 4], output_shape[0].numpy()) - self.assertAllEqual([2, 3], output_shape[1].numpy()) + self.assertAllEqual([[0, 2], [1, 0], [1, 1]], output_indices[0]) + self.assertAllEqual([[0, 0], [0, 1]], output_indices[1]) + self.assertAllEqual([2, 7, 11], output_values[0]) + self.assertAllEqual([3, 5], output_values[1]) + self.assertAllEqual([2, 4], output_shape[0]) + self.assertAllEqual([2, 3], output_shape[1]) # TODO(josh11b): Test an op that has multiple outputs, some but not # all of which are lists. Examples: barrier_take_many (currently @@ -154,84 +154,84 @@ class OpsTest(test_util.TensorFlowTestCase): x = constant_op.constant(1, dtype=dtypes.int32) three_x = x + x + x self.assertEquals(dtypes.int32, three_x.dtype) - self.assertEquals(3, three_x.numpy()) + self.assertAllEqual(3, three_x) def testOperatorOverrides(self): # TODO(henrytan): test with negative number. a = constant_op.constant([1]) b = constant_op.constant([2]) - self.assertAllEqual((-a).numpy(), [-1]) - self.assertAllEqual(abs(b).numpy(), [2]) + self.assertAllEqual((-a), [-1]) + self.assertAllEqual(abs(b), [2]) - self.assertAllEqual((a + b).numpy(), [3]) - self.assertAllEqual((a - b).numpy(), [-1]) - self.assertAllEqual((a * b).numpy(), [2]) - self.assertAllEqual((a * a).numpy(), [1]) + self.assertAllEqual((a + b), [3]) + self.assertAllEqual((a - b), [-1]) + self.assertAllEqual((a * b), [2]) + self.assertAllEqual((a * a), [1]) - self.assertAllEqual((a**b).numpy(), [1]) - self.assertAllEqual((a / b).numpy(), [1 / 2]) - self.assertAllEqual((a / a).numpy(), [1]) - self.assertAllEqual((a % b).numpy(), [1]) + self.assertAllEqual((a**b), [1]) + self.assertAllEqual((a / b), [1 / 2]) + self.assertAllEqual((a / a), [1]) + self.assertAllEqual((a % b), [1]) - self.assertAllEqual((a < b).numpy(), [True]) - self.assertAllEqual((a <= b).numpy(), [True]) - self.assertAllEqual((a > b).numpy(), [False]) - self.assertAllEqual((a >= b).numpy(), [False]) + self.assertAllEqual((a < b), [True]) + self.assertAllEqual((a <= b), [True]) + self.assertAllEqual((a > b), [False]) + self.assertAllEqual((a >= b), [False]) self.assertAllEqual((a == b), False) self.assertAllEqual((a != b), True) - self.assertEqual(1, a[constant_op.constant(0)].numpy()) + self.assertAllEqual(1, a[constant_op.constant(0)]) def test_basic_slice(self): npt = np.arange(1, 19, dtype=np.float32).reshape(3, 2, 3) t = constant_op.constant(npt) - self.assertAllEqual(npt[:, :, :], t[:, :, :].numpy()) - self.assertAllEqual(npt[::, ::, ::], t[::, ::, ::].numpy()) - self.assertAllEqual(npt[::1, ::1, ::1], t[::1, ::1, ::1].numpy()) - self.assertAllEqual(npt[::1, ::5, ::2], t[::1, ::5, ::2].numpy()) - self.assertAllEqual(npt[::-1, :, :], t[::-1, :, :].numpy()) - self.assertAllEqual(npt[:, ::-1, :], t[:, ::-1, :].numpy()) - self.assertAllEqual(npt[:, :, ::-1], t[:, :, ::-1].numpy()) - self.assertAllEqual(npt[-2::-1, :, ::1], t[-2::-1, :, ::1].numpy()) - self.assertAllEqual(npt[-2::-1, :, ::2], t[-2::-1, :, ::2].numpy()) + self.assertAllEqual(npt[:, :, :], t[:, :, :]) + self.assertAllEqual(npt[::, ::, ::], t[::, ::, ::]) + self.assertAllEqual(npt[::1, ::1, ::1], t[::1, ::1, ::1]) + self.assertAllEqual(npt[::1, ::5, ::2], t[::1, ::5, ::2]) + self.assertAllEqual(npt[::-1, :, :], t[::-1, :, :]) + self.assertAllEqual(npt[:, ::-1, :], t[:, ::-1, :]) + self.assertAllEqual(npt[:, :, ::-1], t[:, :, ::-1]) + self.assertAllEqual(npt[-2::-1, :, ::1], t[-2::-1, :, ::1]) + self.assertAllEqual(npt[-2::-1, :, ::2], t[-2::-1, :, ::2]) def testDegenerateSlices(self): npt = np.arange(1, 19, dtype=np.float32).reshape(3, 2, 3) t = constant_op.constant(npt) # degenerate by offering a forward interval with a negative stride - self.assertAllEqual(npt[0:-1:-1, :, :], t[0:-1:-1, :, :].numpy()) + self.assertAllEqual(npt[0:-1:-1, :, :], t[0:-1:-1, :, :]) # degenerate with a reverse interval with a positive stride - self.assertAllEqual(npt[-1:0, :, :], t[-1:0, :, :].numpy()) + self.assertAllEqual(npt[-1:0, :, :], t[-1:0, :, :]) # empty interval in every dimension - self.assertAllEqual(npt[-1:0, 2:2, 2:3:-1], t[-1:0, 2:2, 2:3:-1].numpy()) + self.assertAllEqual(npt[-1:0, 2:2, 2:3:-1], t[-1:0, 2:2, 2:3:-1]) def testEllipsis(self): npt = np.array( [[[[[1, 2], [3, 4], [5, 6]]], [[[7, 8], [9, 10], [11, 12]]]]]) t = constant_op.constant(npt) - self.assertAllEqual(npt[0:], t[0:].numpy()) + self.assertAllEqual(npt[0:], t[0:]) # implicit ellipsis - self.assertAllEqual(npt[0:, ...], t[0:, ...].numpy()) + self.assertAllEqual(npt[0:, ...], t[0:, ...]) # ellipsis alone - self.assertAllEqual(npt[...], t[...].numpy()) + self.assertAllEqual(npt[...], t[...]) # ellipsis at end - self.assertAllEqual(npt[0:1, ...], t[0:1, ...].numpy()) + self.assertAllEqual(npt[0:1, ...], t[0:1, ...]) # ellipsis at begin - self.assertAllEqual(npt[..., 0:1], t[..., 0:1].numpy()) + self.assertAllEqual(npt[..., 0:1], t[..., 0:1]) # ellipsis at middle - self.assertAllEqual(npt[0:1, ..., 0:1], t[0:1, ..., 0:1].numpy()) + self.assertAllEqual(npt[0:1, ..., 0:1], t[0:1, ..., 0:1]) def testShrink(self): npt = np.array([[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]], [[[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]]]]]) t = constant_op.constant(npt) - self.assertAllEqual(npt[:, :, :, :, 3], t[:, :, :, :, 3].numpy()) - self.assertAllEqual(npt[..., 3], t[..., 3].numpy()) - self.assertAllEqual(npt[:, 0], t[:, 0].numpy()) - self.assertAllEqual(npt[:, :, 0], t[:, :, 0].numpy()) + self.assertAllEqual(npt[:, :, :, :, 3], t[:, :, :, :, 3]) + self.assertAllEqual(npt[..., 3], t[..., 3]) + self.assertAllEqual(npt[:, 0], t[:, 0]) + self.assertAllEqual(npt[:, :, 0], t[:, :, 0]) def testOpWithInputsOnDifferentDevices(self): if not context.context().num_gpus(): @@ -242,7 +242,7 @@ class OpsTest(test_util.TensorFlowTestCase): value = constant_op.constant([1., 2.]).as_gpu_tensor() shape = constant_op.constant([2, 1]) reshaped = array_ops.reshape(value, shape) - self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor().numpy()) + self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor()) # And if the shape is in device memory, it should complain # TODO(ashankar): Revisit this - perhaps instead of complaining, @@ -264,7 +264,7 @@ class OpsTest(test_util.TensorFlowTestCase): # The Shape op kernel on GPU places the output in host memory. value = constant_op.constant([1.]).as_gpu_tensor() shape = array_ops.shape(value) - self.assertEquals([1], shape.numpy()) + self.assertEquals([1], shape) def testRandomUniform(self): scalar_shape = constant_op.constant([], dtype=dtypes.int32) @@ -276,8 +276,8 @@ class OpsTest(test_util.TensorFlowTestCase): x = random_ops.random_uniform( scalar_shape, minval=constant_op.constant(5.), maxval=constant_op.constant(6.)) - self.assertLess(x.numpy(), 6) - self.assertGreaterEqual(x.numpy(), 5) + self.assertLess(x, 6) + self.assertGreaterEqual(x, 5) def testArgsToMatchingEagerDefault(self): # Uses default @@ -298,10 +298,10 @@ class OpsTest(test_util.TensorFlowTestCase): flatten_layer = core.Flatten() x = constant_op.constant([[[-10, -20], [-30, -40]], [[10, 20], [30, 40]]]) y = flatten_layer(x) - self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y.numpy()) + self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y) def testIdentity(self): - self.assertEqual(2, array_ops.identity(2).numpy()) + self.assertAllEqual(2, array_ops.identity(2)) def testIncompatibleSetShape(self): x = constant_op.constant(1) diff --git a/tensorflow/python/eager/tape_test.py b/tensorflow/python/eager/tape_test.py index c34f5cffe3..c97cb62125 100644 --- a/tensorflow/python/eager/tape_test.py +++ b/tensorflow/python/eager/tape_test.py @@ -81,8 +81,8 @@ class TapeTest(test.TestCase): tf_e = tf_d + tf_f tf_da, tf_db = gradients_impl.gradients(tf_e, [tf_a, tf_b]) - self.assertAllEqual(da.numpy(), tf_da.eval()) - self.assertAllEqual(db.numpy(), tf_db.eval()) + self.assertAllEqual(da, tf_da.eval()) + self.assertAllEqual(db, tf_db.eval()) def testBasicFunctional(self): @@ -93,7 +93,7 @@ class TapeTest(test.TestCase): aa = constant_op.constant([[1., 0.], [0., 1.]]) bb = constant_op.constant([[1., 2.], [3., 4.]]) da, = backprop.gradients_function(forward, ['a'])(aa, bb) - self.assertAllEqual(da.numpy(), + self.assertAllEqual(da, math_ops.matmul( array_ops.ones_like(aa), array_ops.transpose(bb)).numpy()) @@ -107,7 +107,7 @@ class TapeTest(test.TestCase): aa = constant_op.constant([[1., 0.], [0., 1.]]) bb = constant_op.constant([[1., 2.], [3., 4.]]) da, = backprop.gradients_function(forward, [0])(aa, bb) - self.assertAllEqual(da.numpy(), + self.assertAllEqual(da, math_ops.matmul( array_ops.ones_like(aa), array_ops.transpose(bb)).numpy()) @@ -121,11 +121,11 @@ class TapeTest(test.TestCase): aa = constant_op.constant([[1., 0.], [0., 1.]]) bb = constant_op.constant([[1., 2.], [3., 4.]]) val, (da,) = backprop.val_and_grad_function(forward, ['a'])(aa, bb) - self.assertAllEqual(da.numpy(), + self.assertAllEqual(da, math_ops.matmul( array_ops.ones_like(aa), - array_ops.transpose(bb)).numpy()) - self.assertAllEqual(val.numpy(), forward(aa, bb).numpy()) + array_ops.transpose(bb))) + self.assertAllEqual(val, forward(aa, bb)) def testTwoOutputs(self): @@ -143,8 +143,8 @@ class TapeTest(test.TestCase): tf_rr = 2 * math_ops.reduce_sum(tf_mm) tf_da, tf_db = gradients_impl.gradients(tf_rr, [tf_a, tf_b]) - self.assertAllEqual(da.numpy(), tf_da.eval()) - self.assertAllEqual(db.numpy(), tf_db.eval()) + self.assertAllEqual(da, tf_da.eval()) + self.assertAllEqual(db, tf_db.eval()) def testGcTwoOutputs(self): @@ -155,7 +155,7 @@ class TapeTest(test.TestCase): labels = constant_op.constant([0]) logits = constant_op.constant([[0.0]]) grad, = backprop.gradients_function(fn, [0])(logits, labels) - self.assertAllEqual(grad.numpy(), [[0.0]]) + self.assertAllEqual(grad, [[0.0]]) def testTfTensor(self): @@ -164,7 +164,7 @@ class TapeTest(test.TestCase): t = constant_op.constant(1.0) g, = backprop.gradients_function(fn, [0])(t) - self.assertEqual(g.numpy(), 1.0) + self.assertAllEqual(g, 1.0) def testTapeGC(self): # TODO(apassos) figure out how to test this without using tape internal diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index 953807fc2a..e31c03c08d 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -45,7 +45,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testScalarTensor(self): t = _create_tensor(3, dtype=dtypes.int32) - self.assertEqual(t.numpy(), _create_tensor(np.array(3)).numpy()) + self.assertAllEqual(t, _create_tensor(np.array(3))) self.assertEqual(dtypes.int32, t.dtype) self.assertEqual(0, t.shape.ndims) self.assertAllEqual([], t.shape.as_list()) @@ -85,12 +85,12 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testNumpyValue(self): values = np.array([3.0]) t = _create_tensor(values) - self.assertAllEqual(values, t.numpy()) + self.assertAllEqual(values, t) def testNumpyValueWithCast(self): values = np.array([3.0], dtype=np.float32) t = _create_tensor(values, dtype=dtypes.float64) - self.assertAllEqual(values, t.numpy()) + self.assertAllEqual(values, t) ctx = context.context() # Bad dtype value. with self.assertRaisesRegexp(TypeError, "Invalid dtype argument value"): @@ -100,13 +100,13 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testNumpyOrderHandling(self): n = np.array([[1, 2], [3, 4]], order="F") t = _create_tensor(n) - self.assertAllEqual([[1, 2], [3, 4]], t.numpy()) + self.assertAllEqual([[1, 2], [3, 4]], t) def testTensorAndNumpyMatrix(self): expected = np.array([[1.0, 2.0], [3.0, 4.0]], np.float32) actual = _create_tensor([[1.0, 2.0], [3.0, 4.0]]) - self.assertAllEqual(expected, actual.numpy()) - self.assertEqual(np.float32, actual.numpy().dtype) + self.assertAllEqual(expected, actual) + self.assertEqual(np.float32, actual.dtype) self.assertEqual(dtypes.float32, actual.dtype) self.assertAllEqual([2, 2], actual.shape.as_list()) @@ -140,7 +140,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): t = _create_tensor(np.eye(3)) tensor_str = str(t) self.assertIn("shape=%s, dtype=%s" % (t.shape, t.dtype.name), tensor_str) - self.assertIn(str(t.numpy()), tensor_str) + self.assertIn(str(t), tensor_str) def testMultiLineTensorRepr(self): t = _create_tensor(np.eye(3)) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index a52a0cfc2d..3ac8a0cb6a 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -25,6 +25,7 @@ import re import sys import threading +import numpy as np import six from six.moves import xrange # pylint: disable=redefined-builtin @@ -618,6 +619,9 @@ class _EagerTensorBase(Tensor): """ return self.as_cpu_tensor()._numpy() # pylint: disable=protected-access + def __array__(self): + return np.array(self.numpy()) + def _numpy(self): raise NotImplementedError() -- GitLab From dc27aecb5b3a259bd35d928c643ee1a548279c3a Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Tue, 17 Oct 2017 19:06:38 -0700 Subject: [PATCH 089/573] Remove thread ordering with sleep statements from coordinator_test. PiperOrigin-RevId: 172550053 --- .../python/training/coordinator_test.py | 90 +++++++++++++------ 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/tensorflow/python/training/coordinator_test.py b/tensorflow/python/training/coordinator_test.py index 8f4cae6f06..149d3eed41 100644 --- a/tensorflow/python/training/coordinator_test.py +++ b/tensorflow/python/training/coordinator_test.py @@ -33,21 +33,26 @@ def StopOnEvent(coord, wait_for_stop, set_when_stopped): set_when_stopped.set() -def RaiseInN(coord, n_secs, ex, report_exception): +def RaiseOnEvent(coord, wait_for_stop, set_when_stopped, ex, report_exception): try: - time.sleep(n_secs) + wait_for_stop.wait() raise ex except RuntimeError as e: if report_exception: coord.request_stop(e) else: coord.request_stop(sys.exc_info()) + finally: + if set_when_stopped: + set_when_stopped.set() -def RaiseInNUsingContextHandler(coord, n_secs, ex): +def RaiseOnEventUsingContextHandler(coord, wait_for_stop, set_when_stopped, ex): with coord.stop_on_exception(): - time.sleep(n_secs) + wait_for_stop.wait() raise ex + if set_when_stopped: + set_when_stopped.set() def SleepABit(n_secs, coord=None): @@ -167,80 +172,113 @@ class CoordinatorTest(test.TestCase): def testJoinRaiseReportExcInfo(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() + ev_2 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("First"), False)), - threading.Thread(target=RaiseInN, - args=(coord, 0.05, RuntimeError("Too late"), False))] + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, ev_2, RuntimeError("First"), False)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_2, None, RuntimeError("Too late"), False))] for t in threads: t.start() + + ev_1.set() + with self.assertRaisesRegexp(RuntimeError, "First"): coord.join(threads) def testJoinRaiseReportException(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() + ev_2 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("First"), True)), - threading.Thread(target=RaiseInN, - args=(coord, 0.05, RuntimeError("Too late"), True))] + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, ev_2, RuntimeError("First"), True)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_2, None, RuntimeError("Too late"), True))] for t in threads: t.start() + + ev_1.set() with self.assertRaisesRegexp(RuntimeError, "First"): coord.join(threads) def testJoinIgnoresOutOfRange(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, - errors_impl.OutOfRangeError(None, None, "First"), - True)) + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, + errors_impl.OutOfRangeError(None, None, "First"), + True)) ] for t in threads: t.start() + + ev_1.set() coord.join(threads) def testJoinIgnoresMyExceptionType(self): coord = coordinator.Coordinator(clean_stop_exception_types=(ValueError,)) + ev_1 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, ValueError("Clean stop"), True)) + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, ValueError("Clean stop"), True)) ] for t in threads: t.start() + + ev_1.set() coord.join(threads) def testJoinRaiseReportExceptionUsingHandler(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() + ev_2 = threading.Event() threads = [ - threading.Thread(target=RaiseInNUsingContextHandler, - args=(coord, 0.01, RuntimeError("First"))), - threading.Thread(target=RaiseInNUsingContextHandler, - args=(coord, 0.05, RuntimeError("Too late")))] + threading.Thread( + target=RaiseOnEventUsingContextHandler, + args=(coord, ev_1, ev_2, RuntimeError("First"))), + threading.Thread( + target=RaiseOnEventUsingContextHandler, + args=(coord, ev_2, None, RuntimeError("Too late")))] for t in threads: t.start() + + ev_1.set() with self.assertRaisesRegexp(RuntimeError, "First"): coord.join(threads) def testClearStopClearsExceptionToo(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("First"), True)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, RuntimeError("First"), True)), ] for t in threads: t.start() + with self.assertRaisesRegexp(RuntimeError, "First"): + ev_1.set() coord.join(threads) coord.clear_stop() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("Second"), True)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, RuntimeError("Second"), True)), ] for t in threads: t.start() with self.assertRaisesRegexp(RuntimeError, "Second"): + ev_1.set() coord.join(threads) def testRequestStopRaisesIfJoined(self): -- GitLab From c696dcf24438fdb29394e776f1c865e0167cd368 Mon Sep 17 00:00:00 2001 From: Martin Wicke Date: Tue, 17 Oct 2017 19:30:00 -0700 Subject: [PATCH 090/573] Fix link in audio tutorial (no https for direct download.tensorflow.org links). PiperOrigin-RevId: 172551595 --- tensorflow/docs_src/tutorials/audio_recognition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/tutorials/audio_recognition.md b/tensorflow/docs_src/tutorials/audio_recognition.md index 670e480b12..336f4d9c18 100644 --- a/tensorflow/docs_src/tutorials/audio_recognition.md +++ b/tensorflow/docs_src/tutorials/audio_recognition.md @@ -25,7 +25,7 @@ python tensorflow/examples/speech_commands/train.py ``` The script will start off by downloading the [Speech Commands -dataset](https://download.tensorflow.org/data/speech_commands_v0.01.tar.gz), +dataset](https://storage.cloud.google.com/download.tensorflow.org/data/speech_commands_v0.01.tar.gz), which consists of 65,000 WAVE audio files of people saying thirty different words. This data was collected by Google and released under a CC BY license, and you can help improve it by [contributing five minutes of your own -- GitLab From a8076b9450ca7873592c115841bdebf5f3febf52 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 20:37:27 -0700 Subject: [PATCH 091/573] [XLA] Try to pass layouts through reshapes. For reshapes where the operand and the output have the same rank, try to pass the layout through the reshape. The layout that's already present was presumably assigned for some reason, so it has a good chance of being good. PiperOrigin-RevId: 172555906 --- .../compiler/xla/service/layout_assignment.cc | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc index 2058706f11..7eda7c2284 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/layout_assignment.cc @@ -732,7 +732,8 @@ std::unique_ptr LayoutAssignment::ChooseOperandLayoutFromOutputLayout( // dimension bound is 1 in the operand shape, there may be several such // layouts. So if 'output_layout' is the default layout, try if the // reshape is a bitcast when using the same layout. This may avoid copy - // operations. + // operations. For similar reasons, if the operand and output have the same + // rank, try to match the operand's layout to the output. if (ShapeUtil::TrueRank(operand->shape()) == 1 && ShapeUtil::Rank(instruction->shape()) == 1) { // Don't assign a layout in case of R1 -> effective R1 reshape. @@ -748,6 +749,13 @@ std::unique_ptr LayoutAssignment::ChooseOperandLayoutFromOutputLayout( if (ShapeUtil::ReshapeIsBitcast(operand_shape, output_shape_with_layout)) { return MakeUnique(operand_shape.layout()); } + if (ShapeUtil::Rank(operand_shape) == ShapeUtil::Rank(output_shape)) { + *operand_shape.mutable_layout() = output_layout; + if (ShapeUtil::ReshapeIsBitcast(operand_shape, + output_shape_with_layout)) { + return MakeUnique(output_layout); + } + } auto aligned_operand_shape = ShapeUtil::AlignLayouts(output_shape_with_layout, operand_shape); if (aligned_operand_shape) { @@ -796,7 +804,8 @@ std::unique_ptr LayoutAssignment::ChooseOutputLayoutFromOperandLayout( // dimension bound is 1 in the user shape, there may be several such // layouts. So if 'operand_layout' is the default layout, try if the // reshape is a bitcast when using the same layout. This may avoid copy - // operations. + // operations. For similar reasons, if the operand and output have the same + // rank, try to match the outputs's layout to the operand. if (ShapeUtil::Rank(operand->shape()) == 1 && ShapeUtil::TrueRank(user->shape()) == 1) { // Don't assign a layout in case of R1 -> effective R1 reshape. @@ -812,6 +821,13 @@ std::unique_ptr LayoutAssignment::ChooseOutputLayoutFromOperandLayout( if (ShapeUtil::ReshapeIsBitcast(output_shape, operand_shape_with_layout)) { return MakeUnique(output_shape.layout()); } + if (ShapeUtil::Rank(operand->shape()) == ShapeUtil::Rank(output_shape)) { + *output_shape.mutable_layout() = operand_layout; + if (ShapeUtil::ReshapeIsBitcast(output_shape, + operand_shape_with_layout)) { + return MakeUnique(operand_layout); + } + } auto aligned_user_shape = ShapeUtil::AlignLayouts(operand_shape_with_layout, output_shape); if (aligned_user_shape) { -- GitLab From 6cf5b86c552093114344bde67ca737fd84b8f57e Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Tue, 17 Oct 2017 20:39:11 -0700 Subject: [PATCH 092/573] Include in transpose_functor.h to fix Windows build. PiperOrigin-RevId: 172556044 --- tensorflow/core/kernels/transpose_functor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h index a2eb0263e8..9781fe3b61 100644 --- a/tensorflow/core/kernels/transpose_functor.h +++ b/tensorflow/core/kernels/transpose_functor.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_ #define TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_ +#include #include #include #include "tensorflow/core/framework/tensor.h" -- GitLab From 35debbdff9e61edc7266bd0ea0636dd5c328114a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 17 Oct 2017 22:40:07 -0700 Subject: [PATCH 093/573] Update inception score to match the openAI version from https://github.com/openai/improved-gan/tree/master/inception_score. PiperOrigin-RevId: 172562573 --- .../eval/python/classifier_metrics_impl.py | 81 +++++++++---------- .../eval/python/classifier_metrics_test.py | 10 +-- 2 files changed, 43 insertions(+), 48 deletions(-) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py index 6074694f8b..4af87b8b47 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py @@ -16,6 +16,11 @@ These methods come from https://arxiv.org/abs/1606.03498 and https://arxiv.org/abs/1706.08500. + +NOTE: This implementation uses the same weights as in +https://github.com/openai/improved-gan/blob/master/inception_score/model.py, +but is more numerically stable and is an unbiased estimator of the true +Inception score even when splitting the inputs into batches. """ from __future__ import absolute_import @@ -54,17 +59,16 @@ __all__ = [ 'classifier_score', 'frechet_inception_distance', 'frechet_classifier_distance', + 'INCEPTION_DEFAULT_IMAGE_SIZE', ] -INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v3_2017_09_13.tar.gz' -INCEPTION_FROZEN_GRAPH = 'frozen_inception_v3.pb' -INCEPTION_V3_INPUT = 'input' -INCEPTION_V3_OUTPUT = 'InceptionV3/Logits/SpatialSqueeze:0' -INCEPTION_V3_FINAL_POOL = 'InceptionV3/Logits/AvgPool_1a_8x8/AvgPool:0' -_INCEPTION_V3_NUM_CLASSES = 1001 -_INCEPTION_V3_FINAL_POOL_SIZE = 2048 -INCEPTION_V3_DEFAULT_IMG_SIZE = 299 +INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v1_2015_12_05.tar.gz' +INCEPTION_FROZEN_GRAPH = 'inceptionv1_for_inception_score.pb' +INCEPTION_INPUT = 'Mul:0' +INCEPTION_OUTPUT = 'logits:0' +INCEPTION_FINAL_POOL = 'pool_3:0' +INCEPTION_DEFAULT_IMAGE_SIZE = 299 def _validate_images(images, image_size): @@ -106,42 +110,33 @@ def _symmetric_matrix_square_root(mat, eps=1e-10): # NOTE: Floating-point inputs are expected to be in [0, 1]. # Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py. def preprocess_image( - image, height=INCEPTION_V3_DEFAULT_IMG_SIZE, - width=INCEPTION_V3_DEFAULT_IMG_SIZE, central_fraction=0.875, scope=None): - """Prepare one image for evaluation. - - If height and width are specified it would output an image with that size by - applying resize_bilinear. + images, height=INCEPTION_DEFAULT_IMAGE_SIZE, + width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None): + """Prepare a batch of images for evaluation. - If central_fraction is specified it would crop the central fraction of the - input image. + This is the preprocessing portion of the graph from + http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz. Args: - image: 3-D Tensor of image. If dtype is tf.float32 then the range should be - [0, 1], otherwise it would converted to tf.float32 assuming that the range - is [0, MAX], where MAX is largest positive representable number for - int(8/16/32) data type (see `tf.image.convert_image_dtype` for details). - height: integer - width: integer - central_fraction: Optional Float, fraction of the image to crop. + images: 3-D or 4-D Tensor of images. Values are in [0, 255]. + height: Integer. Height of resized output image. + width: Integer. Width of resized output image. scope: Optional scope for name_scope. + Returns: - 3-D float Tensor of prepared image. + 3-D or 4-D float Tensor of prepared image(s). Values are in [-1, 1]. """ - with ops.name_scope(scope, 'eval_image', [image, height, width]): - if image.dtype != dtypes.float32: - image = image_ops.convert_image_dtype(image, dtype=dtypes.float32) - # Crop the central region of the image with an area containing 87.5% of - # the original image. - image = image_ops.central_crop(image, central_fraction=central_fraction) - - # Resize the image to the specified height and width. - image = array_ops.expand_dims(image, 0) - image = image_ops.resize_bilinear(image, [height, width], - align_corners=False) - image = array_ops.squeeze(image, [0]) - image = (image - 0.5) * 2.0 - return image + is_single = images.shape.ndims == 3 + with ops.name_scope(scope, 'preprocess', [images, height, width]): + if not images.dtype.is_floating: + images = math_ops.to_float(images) + images = (images - 128.0) / 128.0 + if is_single: + images = array_ops.expand_dims(images, axis=0) + resized = image_ops.resize_bilinear(images, [height, width]) + if is_single: + resized = array_ops.squeeze(resized, axis=0) + return resized def _kl_divergence(p, p_logits, q): @@ -211,9 +206,9 @@ def _default_graph_def_fn(): def run_inception(images, graph_def=None, default_graph_def_fn=_default_graph_def_fn, - image_size=INCEPTION_V3_DEFAULT_IMG_SIZE, - input_tensor=INCEPTION_V3_INPUT, - output_tensor=INCEPTION_V3_OUTPUT): + image_size=INCEPTION_DEFAULT_IMAGE_SIZE, + input_tensor=INCEPTION_INPUT, + output_tensor=INCEPTION_OUTPUT): """Run images through a pretrained Inception classifier. Args: @@ -338,7 +333,7 @@ def classifier_score(images, classifier_fn, num_batches=1): inception_score = functools.partial( classifier_score, classifier_fn=functools.partial( - run_inception, output_tensor=INCEPTION_V3_OUTPUT)) + run_inception, output_tensor=INCEPTION_OUTPUT)) def trace_sqrt_product(sigma, sigma_v): @@ -479,4 +474,4 @@ def frechet_classifier_distance(real_images, frechet_inception_distance = functools.partial( frechet_classifier_distance, classifier_fn=functools.partial( - run_inception, output_tensor=INCEPTION_V3_FINAL_POOL)) + run_inception, output_tensor=INCEPTION_FINAL_POOL)) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py index 30285964a5..81fa2fc0f1 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py @@ -68,7 +68,7 @@ def _expected_trace_sqrt_product(sigma, sigma_v): # A dummy GraphDef string with the minimum number of Ops. graphdef_string = """ node { - name: "input" + name: "Mul" op: "Placeholder" attr { key: "dtype" @@ -97,7 +97,7 @@ node { } } node { - name: "InceptionV3/Logits/SpatialSqueeze" + name: "logits" op: "Placeholder" attr { key: "dtype" @@ -120,7 +120,7 @@ node { } } node { - name: "InceptionV3/Logits/AvgPool_1a_8x8/AvgPool" + name: "pool_3" op: "Placeholder" attr { key: "dtype" @@ -182,7 +182,7 @@ class ClassifierMetricsTest(test.TestCase): img = array_ops.ones([batch_size, 299, 299, 3]) pool = _run_with_mock( classifier_metrics.run_inception, img, - output_tensor=classifier_metrics.INCEPTION_V3_FINAL_POOL) + output_tensor=classifier_metrics.INCEPTION_FINAL_POOL) self.assertTrue(isinstance(pool, ops.Tensor)) pool.shape.assert_is_compatible_with([batch_size, 2048]) @@ -306,7 +306,7 @@ class ClassifierMetricsTest(test.TestCase): """Test `preprocess_image` graph construction.""" incorrectly_sized_image = array_ops.zeros([520, 240, 3]) correct_image = classifier_metrics.preprocess_image( - image=incorrectly_sized_image) + images=incorrectly_sized_image) _run_with_mock(classifier_metrics.run_inception, array_ops.expand_dims(correct_image, 0)) -- GitLab From bedfe8ac14bddbf21c5acf80d55abff9df4a7967 Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Wed, 18 Oct 2017 08:46:24 +0300 Subject: [PATCH 094/573] Fix minor typos in TF Boosted Trees (#13792) * Fix a typo in readme of a module of boosted trees * Remove trailing backslashes --- tensorflow/contrib/boosted_trees/README.md | 2 +- tensorflow/contrib/boosted_trees/examples/binary_mnist.py | 2 +- tensorflow/contrib/boosted_trees/examples/mnist.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md index 9ce700f1a1..7d30032e53 100644 --- a/tensorflow/contrib/boosted_trees/README.md +++ b/tensorflow/contrib/boosted_trees/README.md @@ -1,7 +1,7 @@ # TF Boosted Trees (TFBT) TF Boosted trees is an implementation of a gradient boosting algorithm with -trees used as week learners. +trees used as weak learners. ## Examples Folder "examples" demonstrates how TFBT estimators can be used for various diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py index c003b1de66..47ee3d816f 100644 --- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py +++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py @@ -21,7 +21,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example. python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \ --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \ --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \ - --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 \ + --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 When training is done, accuracy on eval data is reported. Point tensorboard to the directory for the run to see how the training progresses: diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py index 0539d77720..817c6eb3e1 100644 --- a/tensorflow/contrib/boosted_trees/examples/mnist.py +++ b/tensorflow/contrib/boosted_trees/examples/mnist.py @@ -22,7 +22,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example. python tensorflow/contrib/boosted_trees/examples/mnist.py \ --output_dir="/tmp/mnist" --depth=4 --learning_rate=0.3 --batch_size=60000 \ --examples_per_layer=60000 --eval_batch_size=10000 --num_eval_steps=1 \ - --num_trees=10 --l2=1 --vmodule=training_ops=1 \ + --num_trees=10 --l2=1 --vmodule=training_ops=1 When training is done, accuracy on eval data is reported. Point tensorboard to the directory for the run to see how the training progresses: -- GitLab From 130ec39dae7bc7e71b739520ea65689f63e292d6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 05:04:57 -0700 Subject: [PATCH 095/573] Stub support for retrieving LossFunction by name. PiperOrigin-RevId: 172588516 --- .../kernel_tests/layer_collection_test.py | 67 +++++++++++++++++++ .../python/kernel_tests/optimizer_test.py | 6 +- .../kfac/python/ops/layer_collection.py | 58 +++++++++++++--- 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 633104ace0..13c69d261c 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -30,6 +30,43 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test +class LayerParametersDictTest(test.TestCase): + + def testSetItem(self): + """Ensure insertion, contains, retrieval works for supported key types.""" + with ops.Graph().as_default(): + lp_dict = layer_collection.LayerParametersDict() + + x = array_ops.constant(0) + y0 = array_ops.constant(0) + y1 = array_ops.constant(0) + z0 = array_ops.constant(0) + z1 = array_ops.constant(0) + keys = [x, (y0, y1), [z0, z1]] + for key in keys: + lp_dict[key] = key + + for key in keys: + self.assertTrue(key in lp_dict) + self.assertEqual(lp_dict[key], key) + + def testSetItemOverlap(self): + """Ensure insertion fails if key overlaps with existing key.""" + with ops.Graph().as_default(): + lp_dict = layer_collection.LayerParametersDict() + + x = array_ops.constant(0) + y = array_ops.constant(0) + lp_dict[x] = 'value' + + with self.assertRaises(ValueError): + lp_dict[(x, y)] = 'value' + + # Ensure 'y' wasn't inserted. + self.assertTrue(x in lp_dict) + self.assertFalse(y in lp_dict) + + class LayerCollectionTest(test.TestCase): def testLayerCollectionInit(self): @@ -157,6 +194,36 @@ class LayerCollectionTest(test.TestCase): double_loss = sess.run(lc2.total_sampled_loss()) self.assertAlmostEqual(2 * single_loss, double_loss) + def testLossFunctionByName(self): + """Ensure loss functions can be identified by name.""" + with ops.Graph().as_default(): + logits = linalg_ops.eye(2) + lc = layer_collection.LayerCollection() + + # Create a new loss function by name. + lc.register_categorical_predictive_distribution(logits, name='loss1') + self.assertEqual(1, len(lc.losses)) + + # Add logits to same loss function. + with self.assertRaises(NotImplementedError): + lc.register_categorical_predictive_distribution(logits, name='loss1') + self.assertEqual(1, len(lc.losses)) + + # Add another new loss function. + lc.register_categorical_predictive_distribution(logits, name='loss2') + self.assertEqual(2, len(lc.losses)) + + def testLossFunctionWithoutName(self): + """Ensure loss functions get unique names if 'name' not specified.""" + with ops.Graph().as_default(): + logits = linalg_ops.eye(2) + lc = layer_collection.LayerCollection() + + # Create a new loss function by name. + lc.register_categorical_predictive_distribution(logits) + lc.register_categorical_predictive_distribution(logits) + self.assertEqual(2, len(lc.losses)) + def testRegisterCategoricalPredictiveDistributionBatchSize1(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py index 5f28f57f6a..9325aa1b73 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import numpy as np from tensorflow.contrib.kfac.python.ops import layer_collection as lc -from tensorflow.contrib.kfac.python.ops import loss_functions as lf from tensorflow.contrib.kfac.python.ops import optimizer from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -124,9 +123,8 @@ class OptimizerTest(test.TestCase): def testUpdateVelocities(self): with ops.Graph().as_default(), self.test_session() as sess: layers = lc.LayerCollection() - layers.losses = [ - lf.CategoricalLogitsNegativeLogProbLoss(array_ops.constant([1.0])) - ] + layers.register_categorical_predictive_distribution( + array_ops.constant([1.0])) opt = optimizer.KfacOptimizer( 0.1, 0.2, 0.3, layers, momentum=0.5, momentum_type='regular') x = variable_scope.get_variable('x', initializer=array_ops.ones((2, 2))) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 1b77f5d3ba..0cb55894ad 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -55,6 +55,7 @@ class LayerParametersDict(OrderedDict): super(LayerParametersDict, self).__init__(*args, **kwargs) def __setitem__(self, key, value): + key = self._canonicalize_key(key) tensors = key if isinstance(key, (tuple, list)) else (key,) key_collisions = self._tensors.intersection(tensors) if key_collisions: @@ -63,9 +64,23 @@ class LayerParametersDict(OrderedDict): super(LayerParametersDict, self).__setitem__(key, value) def __delitem__(self, key): + key = self._canonicalize_key(key) self._tensors.remove(key) super(LayerParametersDict, self).__delitem__(key) + def __getitem__(self, key): + key = self._canonicalize_key(key) + return super(LayerParametersDict, self).__getitem__(key) + + def __contains__(self, key): + key = self._canonicalize_key(key) + return super(LayerParametersDict, self).__contains__(key) + + def _canonicalize_key(self, key): + if isinstance(key, (list, tuple)): + return tuple(key) + return key + # TODO(duckworthd): add capability for LayerCollection to be "finalized" # and do this when it gets used by FisherEstimator / KfacOptimizer @@ -94,13 +109,16 @@ class LayerCollection(object): self.fisher_factors = OrderedDict() self._generic_registrations = set() self._graph = graph or ops.get_default_graph() - self.losses = [] + self._loss_dict = {} # {str: LossFunction} self._subgraph = None with variable_scope.variable_scope(None, default_name=name) as scope: self._var_scope = scope.name - reset_internals = __init__ + @property + def losses(self): + """LossFunctions registered with this LayerCollection.""" + return list(self._loss_dict.values()) def register_block(self, layer_key, fisher_block): """Validates and registers the layer_key associated with the fisher_block. @@ -277,7 +295,8 @@ class LayerCollection(object): def register_categorical_predictive_distribution(self, logits, seed=None, - targets=None): + targets=None, + name=None): """Registers a categorical predictive distribution. Args: @@ -288,16 +307,24 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_categorical_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.CategoricalLogitsNegativeLogProbLoss( logits, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def register_normal_predictive_distribution(self, mean, var=0.5, seed=None, - targets=None): + targets=None, + name=None): """Registers a normal predictive distribution. Args: @@ -312,15 +339,23 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_normal_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.NormalMeanNegativeLogProbLoss( mean, var, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def register_multi_bernoulli_predictive_distribution(self, logits, seed=None, - targets=None): + targets=None, + name=None): """Registers a multi-Bernoulli predictive distribution. Args: @@ -331,10 +366,17 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_multi_bernoulli_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.MultiBernoulliNegativeLogProbLoss( logits, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def make_or_get_factor(self, cls, args): with variable_scope.variable_scope(self._var_scope): -- GitLab From 139e1e0771faeaa614e3c6672a5c203866ba0176 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 18 Oct 2017 09:18:00 -0700 Subject: [PATCH 096/573] Add `int64` axis support for `tf.cumsum` and `tf.cumprod` (#13791) * Add `int64` axis support for `tf.cumsum` and `tf.cumprod` This fix adds `int64` axis support for `tf.cumsum` and `tf.cumprod`. Though `int64` is the registered data type for `axis` (`Tidx`), no kernel is available. The issue could be described as: ``` >>> import tensorflow as tf >>> v = tf.cumsum([1, 2, 3], tf.constant(0, tf.int64)) >>> tf.Session().run(v) Traceback (most recent call last): File "", line 1, in File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 889, in run run_metadata_ptr) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1120, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1317, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1336, in _do_call raise type(e)(node_def, op, message) tensorflow.python.framework.errors_impl.InvalidArgumentError: No OpKernel was registered to support Op 'Cumsum' with these attrs. Registered devices: [CPU], Registered kernels: device='CPU'; T in [DT_COMPLEX128]; Tidx in [DT_INT32] device='CPU'; T in [DT_COMPLEX64]; Tidx in [DT_INT32] device='CPU'; T in [DT_DOUBLE]; Tidx in [DT_INT32] device='CPU'; T in [DT_FLOAT]; Tidx in [DT_INT32] device='CPU'; T in [DT_HALF]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT8]; Tidx in [DT_INT32] device='CPU'; T in [DT_UINT8]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT16]; Tidx in [DT_INT32] device='CPU'; T in [DT_UINT16]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT32]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT64]; Tidx in [DT_INT32] [[Node: Cumsum = Cumsum[T=DT_INT32, Tidx=DT_INT64, exclusive=false, reverse=false](Cumsum/x, Const)]] Caused by op u'Cumsum', defined at: File "", line 1, in File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py", line 2246, in cumsum x, axis, exclusive=exclusive, reverse=reverse, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 1370, in cumsum name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2966, in create_op op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1473, in __init__ self._traceback = self._graph._extract_stack() # pylint: disable=protected-access InvalidArgumentError (see above for traceback): No OpKernel was registered to support Op 'Cumsum' with these attrs. Registered devices: [CPU], Registered kernels: device='CPU'; T in [DT_COMPLEX128]; Tidx in [DT_INT32] device='CPU'; T in [DT_COMPLEX64]; Tidx in [DT_INT32] device='CPU'; T in [DT_DOUBLE]; Tidx in [DT_INT32] device='CPU'; T in [DT_FLOAT]; Tidx in [DT_INT32] device='CPU'; T in [DT_HALF]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT8]; Tidx in [DT_INT32] device='CPU'; T in [DT_UINT8]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT16]; Tidx in [DT_INT32] device='CPU'; T in [DT_UINT16]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT32]; Tidx in [DT_INT32] device='CPU'; T in [DT_INT64]; Tidx in [DT_INT32] [[Node: Cumsum = Cumsum[T=DT_INT32, Tidx=DT_INT64, exclusive=false, reverse=false](Cumsum/x, Const)]] >>> ``` This fix adds the missing kernel. Signed-off-by: Yong Tang * Add test cases for `int64` axis support of `tf.cumsum` and `tf.cumprod` Signed-off-by: Yong Tang * Reformat scan_ops.cc with `clang-format -i` Signed-off-by: Yong Tang --- tensorflow/core/kernels/scan_ops.cc | 98 ++++++++++++------- .../python/kernel_tests/scan_ops_test.py | 18 ++++ 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc index cc434ab0ae..0a6848361a 100644 --- a/tensorflow/core/kernels/scan_ops.cc +++ b/tensorflow/core/kernels/scan_ops.cc @@ -35,7 +35,7 @@ namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; -template +template class ScanOp : public OpKernel { public: explicit ScanOp(OpKernelConstruction* ctx) : OpKernel(ctx) { @@ -51,8 +51,9 @@ class ScanOp : public OpKernel { errors::InvalidArgument("ScanOp: axis must be a scalar, not ", tensor_axis.shape().DebugString())); - const int axis_arg = internal::SubtleMustCopy(tensor_axis.scalar()()); - const int axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg; + const Tidx axis_arg = + internal::SubtleMustCopy(tensor_axis.scalar()()); + const Tidx axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg; OP_REQUIRES(ctx, FastBoundsCheck(axis, input.dims()), errors::InvalidArgument( "ScanOp: Expected scan axis in the range [", -input.dims(), @@ -70,11 +71,11 @@ class ScanOp : public OpKernel { // Dim reduction. int64 reduced_shape[3] = {1, 1, 1}; - for (int i = 0; i < axis; ++i) { + for (Tidx i = 0; i < axis; ++i) { reduced_shape[0] *= input.dim_size(i); } reduced_shape[1] = input.dim_size(axis); - for (int i = axis + 1; i < input.dims(); ++i) { + for (Tidx i = axis + 1; i < input.dims(); ++i) { reduced_shape[2] *= input.dim_size(i); } @@ -112,51 +113,76 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_FOR_ALL_REDUCERS); } // namespace functor #endif // GOOGLE_CUDA - // Register Cumsum kernels -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumsum") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ScanOp>) +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int64>) TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumsum") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("axis"), \ - ScanOp>) +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int64>) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS) #undef REGISTER_GPU_KERNELS #endif // GOOGLE_CUDA // Register Cumprod kernels -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumprod") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ScanOp>) +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int64>) TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumprod") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("axis"), \ - ScanOp>) +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int64>) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS) #undef REGISTER_GPU_KERNELS #endif // GOOGLE_CUDA diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py index 6b2b589a06..08b4a2aaae 100644 --- a/tensorflow/python/kernel_tests/scan_ops_test.py +++ b/tensorflow/python/kernel_tests/scan_ops_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import gradient_checker @@ -92,6 +94,14 @@ class CumsumTest(test.TestCase): for axis in (-1, 0): self._compareAll(x, axis) + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True): + axis = constant_op.constant(0, axis_dtype) + tf_out = math_ops.cumsum(x, axis).eval() + def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) @@ -190,6 +200,14 @@ class CumprodTest(test.TestCase): for axis in (-1, 0): self._compareAll(x, axis) + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True): + axis = constant_op.constant(0, axis_dtype) + tf_out = math_ops.cumprod(x, axis).eval() + def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) -- GitLab From 75da6f62494e1fd6dd7c197e7b5b79a1a451fb3d Mon Sep 17 00:00:00 2001 From: Christian Grail Date: Wed, 18 Oct 2017 18:22:45 +0200 Subject: [PATCH 097/573] gitignore: ignore build files relevant for iOS sample apps (#13809) * gitignore: ignore build files relavant for ios sample apps * Add extra linespace at the end --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 9572a3e97c..9ae0d9c96f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ cmake_build/ /build/ /tensorflow/core/util/version_info.cc /tensorflow/python/framework/fast_tensor_util.cpp +Pods +Podfile.lock +*.pbxproj +*.xcworkspacedata -- GitLab From 3c31886537a8b5fb5ab62b4b925f8ef044960ca3 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Wed, 18 Oct 2017 09:38:03 -0700 Subject: [PATCH 098/573] Don't emit fusion computations separately in HloModule::ToString. These computations are emitted with their fusion instruction and therefore don't need to be emitted as a separate comptutation in the module. PiperOrigin-RevId: 172612725 --- tensorflow/compiler/xla/service/hlo_module.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 5bc7a36439..9d4a994838 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -154,8 +154,8 @@ string HloModule::ToString() const { std::ostringstream s; s << "HloModule " << name() << ":\n\n"; s << "ENTRY " << entry_computation()->ToString() << "\n\n"; - for (const std::unique_ptr& computation : computations_) { - if (computation.get() != entry_computation()) { + for (const HloComputation* computation : MakeNonfusionComputations()) { + if (computation != entry_computation()) { s << computation->ToString() << "\n\n"; } } -- GitLab From c9d3377fb4f973e1592ebc71862e02dacf5f3a4f Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Wed, 18 Oct 2017 10:49:20 -0700 Subject: [PATCH 099/573] Make `tf.contrib.distributions` quadrature family parameterized by `quadrature_grid_and_prob` vs `quadrature_degree`. Enables support of quadrature methods other than Gauss-Hermite. PiperOrigin-RevId: 172622919 --- .../kernel_tests/poisson_lognormal_test.py | 20 +++++-- .../python/ops/poisson_lognormal.py | 54 +++++++++++------- .../python/ops/vector_diffeomixture.py | 55 +++++++++++-------- 3 files changed, 79 insertions(+), 50 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py index 7cb46bb236..3ded4159d8 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import numpy as np + from tensorflow.contrib.distributions.python.ops import poisson_lognormal from tensorflow.contrib.distributions.python.ops import test_util from tensorflow.python.platform import test @@ -32,7 +34,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=-2., scale=1.1, - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( sess, pln, rtol=0.1) @@ -42,7 +45,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=0., scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( sess, pln, rtol=0.02) @@ -52,7 +56,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( sess, pln, rtol=0.1, atol=0.01) @@ -62,7 +67,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( sess, pln, rtol=0.1, atol=0.01) @@ -72,7 +78,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[[0.], [-0.5]], scale=[[1., 0.9]], - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( sess, pln, rtol=0.1, atol=0.08) @@ -82,7 +89,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[[0.], [-0.5]], scale=[[1., 0.9]], - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( sess, pln, rtol=0.1, atol=0.01) diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 65ee3a16d6..80d4e2dc5e 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -93,7 +93,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): : d=0, ..., deg-1 } ``` - where, [`grid, w = numpy.polynomial.hermite.hermgauss(deg)`]( + where, [e.g., `grid, w = numpy.polynomial.hermite.hermgauss(deg)`]( https://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.polynomial.hermite.hermgauss.html) and `prob = w / sqrt(pi)`. @@ -106,14 +106,15 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): pln = ds.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) """ def __init__(self, loc, scale, - quadrature_polynomial_degree=8, + quadrature_grid_and_probs=None, validate_args=False, allow_nan_stats=True, name="PoissonLogNormalQuadratureCompound"): @@ -124,8 +125,9 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): the LogNormal prior. scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of the LogNormal prior. - quadrature_polynomial_degree: Python `int`-like scalar. - Default value: 8. + quadrature_grid_and_probs: Python pair of `list`-like objects representing + the sample points and the corresponding (possibly normalized) weight. + When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -138,6 +140,8 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): Raises: TypeError: if `loc.dtype != scale[0].dtype`. + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` """ parameters = locals() with ops.name_scope(name, values=[loc, scale]): @@ -153,18 +157,21 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format( loc.dtype.name, scale.dtype.name)) - self._degree = quadrature_polynomial_degree - - grid, prob = np.polynomial.hermite.hermgauss( - deg=quadrature_polynomial_degree) - - # It should be that `sum(prob) == sqrt(pi)`, but self-normalization is - # more numerically stable. - prob = prob.astype(dtype.as_numpy_dtype) - prob /= np.linalg.norm(prob, ord=1) + if quadrature_grid_and_probs is None: + grid, probs = np.polynomial.hermite.hermgauss(deg=8) + else: + grid, probs = tuple(quadrature_grid_and_probs) + if len(grid) != len(probs): + raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length list-like objects") + grid = grid.astype(dtype.as_numpy_dtype) + probs = probs.astype(dtype.as_numpy_dtype) + probs /= np.linalg.norm(probs, ord=1) + self._quadrature_grid = grid + self._quadrature_probs = probs self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(prob), + logits=np.log(probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -210,9 +217,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): return self._scale @property - def quadrature_polynomial_degree(self): - """Polynomial largest exponent used for Gauss-Hermite quadrature.""" - return self._degree + def quadrature_grid(self): + """Quadrature grid points.""" + return self._quadrature_grid + + @property + def quadrature_probs(self): + """Quadrature normalized weights.""" + return self._quadrature_probs def _batch_shape_tensor(self): return array_ops.broadcast_dynamic_shape( @@ -242,10 +254,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): [batch_size])), seed=distribution_util.gen_new_seed( seed, "poisson_lognormal_quadrature_compound")) - # Stride `quadrature_polynomial_degree` for `batch_size` number of times. + # Stride `quadrature_degree` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * self._degree, - delta=self._degree, + limit=batch_size * len(self.quadrature_probs), + delta=len(self.quadrature_probs), dtype=ids.dtype) ids += offset rate = array_ops.gather( diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py index 438d628da4..33dad811a9 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py @@ -141,7 +141,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): and, ```none - grid, weight = np.polynomial.hermite.hermgauss(quadrature_polynomial_degree) + grid, weight = np.polynomial.hermite.hermgauss(quadrature_degree) prob[k] = weight[k] / sqrt(pi) lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i]) ``` @@ -219,7 +219,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): distribution, loc=None, scale=None, - quadrature_polynomial_degree=8, + quadrature_grid_and_probs=None, validate_args=False, allow_nan_stats=True, name="VectorDiffeomixture"): @@ -248,7 +248,9 @@ class VectorDiffeomixture(distribution_lib.Distribution): `k`-th element represents the `scale` used for the `k`-th affine transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`, `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices - quadrature_polynomial_degree: Python `int`-like scalar. + quadrature_grid_and_probs: Python pair of `list`-like objects representing + the sample points and the corresponding (possibly normalized) weight. + When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -262,7 +264,8 @@ class VectorDiffeomixture(distribution_lib.Distribution): Raises: ValueError: if `not scale or len(scale) < 2`. ValueError: if `len(loc) != len(scale)` - ValueError: if `quadrature_polynomial_degree < 1`. + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` ValueError: if `validate_args` and any not scale.is_positive_definite. TypeError: if any scale.dtype != scale[0].dtype. TypeError: if any loc.dtype != scale[0].dtype. @@ -307,12 +310,6 @@ class VectorDiffeomixture(distribution_lib.Distribution): name="endpoint_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip(loc, scale))] - if quadrature_polynomial_degree < 1: - raise ValueError("quadrature_polynomial_degree={} " - "is not at least 1".format( - quadrature_polynomial_degree)) - self._degree = quadrature_polynomial_degree - # TODO(jvdillon): Remove once we support k-mixtures. # We make this assertion here because otherwise `grid` would need to be a # vector not a scalar. @@ -320,17 +317,24 @@ class VectorDiffeomixture(distribution_lib.Distribution): raise NotImplementedError("Currently only bimixtures are supported; " "len(scale)={} is not 2.".format(len(scale))) - grid, prob = np.polynomial.hermite.hermgauss( - deg=quadrature_polynomial_degree) + if quadrature_grid_and_probs is None: + grid, probs = np.polynomial.hermite.hermgauss(deg=8) + else: + grid, probs = tuple(quadrature_grid_and_probs) + if len(grid) != len(probs): + raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length list-like objects") grid = grid.astype(dtype.as_numpy_dtype) - prob = prob.astype(dtype.as_numpy_dtype) - prob /= np.linalg.norm(prob, ord=1) + probs = probs.astype(dtype.as_numpy_dtype) + probs /= np.linalg.norm(probs, ord=1) + self._quadrature_grid = grid + self._quadrature_probs = probs # Note: by creating the logits as `log(prob)` we ensure that # `self.mixture_distribution.logits` is equivalent to # `math_ops.log(self.mixture_distribution.probs)`. self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(prob), + logits=np.log(probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -357,10 +361,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): validate_args=validate_args, name="interpolated_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip( - interpolate_loc(quadrature_polynomial_degree, + interpolate_loc(len(self._quadrature_grid), self._interpolate_weight, loc), - interpolate_scale(quadrature_polynomial_degree, + interpolate_scale(len(self._quadrature_grid), self._interpolate_weight, scale)))] @@ -416,9 +420,14 @@ class VectorDiffeomixture(distribution_lib.Distribution): return self._interpolated_affine @property - def quadrature_polynomial_degree(self): - """Polynomial largest exponent used for Gauss-Hermite quadrature.""" - return self._degree + def quadrature_grid(self): + """Quadrature grid points.""" + return self._quadrature_grid + + @property + def quadrature_probs(self): + """Quadrature normalized weights.""" + return self._quadrature_probs def _batch_shape_tensor(self): return self._batch_shape_ @@ -454,10 +463,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): seed=distribution_util.gen_new_seed( seed, "vector_diffeomixture")) - # Stride `self._degree` for `batch_size` number of times. + # Stride `quadrature_degree` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * self._degree, - delta=self._degree, + limit=batch_size * len(self.quadrature_probs), + delta=len(self.quadrature_probs), dtype=ids.dtype) weight = array_ops.gather( -- GitLab From bc0822675598385d8068bf114b453dac52512caf Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 18 Oct 2017 11:09:27 -0700 Subject: [PATCH 100/573] Fixes test breakage. PiperOrigin-RevId: 172626499 --- tensorflow/python/eager/BUILD | 7 +++---- tensorflow/python/eager/ops_test.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 9e9a7f4c59..ef04f933c5 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -391,14 +391,14 @@ py_test( ], ) -py_test( +cuda_py_test( name = "ops_test", srcs = ["ops_test.py"], - srcs_version = "PY2AND3", - deps = [ + additional_deps = [ ":context", ":execute", ":test", + "//third_party/py/numpy", "//tensorflow/python:array_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", @@ -410,7 +410,6 @@ py_test( "//tensorflow/python:random_ops", "//tensorflow/python:sparse_ops", "//tensorflow/python:tensor_shape", - "//third_party/py/numpy", ], ) diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 6d1a5fe264..f737bfbc15 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -49,7 +49,7 @@ class OpsTest(test_util.TensorFlowTestCase): three = constant_op.constant([[3.]]).as_gpu_tensor() five = constant_op.constant([[5.]]).as_gpu_tensor() product = math_ops.matmul(three, five) - self.assertEqual([[15.0]], product) + self.assertEqual([[15.0]], product.numpy()) def testExecuteStringAttr(self): three = constant_op.constant(3.0) @@ -97,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase): self.skipTest('No GPUs found') with context.device('/gpu:0'): r = constant_op.constant(1) + constant_op.constant(2) - self.assertEqual(r, 3) + self.assertAllEqual(r, 3) def testExecuteListOutputLen1(self): split_dim = constant_op.constant(1) @@ -264,7 +264,7 @@ class OpsTest(test_util.TensorFlowTestCase): # The Shape op kernel on GPU places the output in host memory. value = constant_op.constant([1.]).as_gpu_tensor() shape = array_ops.shape(value) - self.assertEquals([1], shape) + self.assertEqual([1], shape.numpy()) def testRandomUniform(self): scalar_shape = constant_op.constant([], dtype=dtypes.int32) -- GitLab From 9aadd980cd0202a338e058da05c9970ccb160bc5 Mon Sep 17 00:00:00 2001 From: Jinze Bai Date: Thu, 19 Oct 2017 02:22:24 +0800 Subject: [PATCH 101/573] Add nth_element op (#13720) * add nth_element op * change op order in buildifier * remove the symbol in ops/nn.py * add nth_element symbol in contrib.nn * change nth_element symbol to ops.nn_ops --- tensorflow/contrib/nn/__init__.py | 2 + tensorflow/core/kernels/BUILD | 7 + tensorflow/core/kernels/nth_element_op.cc | 139 ++++++++++++++ tensorflow/core/kernels/nth_element_op.h | 39 ++++ tensorflow/core/ops/nn_ops.cc | 50 +++++ tensorflow/core/ops/nn_ops_test.cc | 24 +++ tensorflow/python/kernel_tests/BUILD | 15 ++ .../kernel_tests/nth_element_op_test.py | 174 ++++++++++++++++++ tensorflow/python/ops/nn_grad.py | 29 +++ tensorflow/python/ops/nn_ops.py | 28 +++ 10 files changed, 507 insertions(+) create mode 100644 tensorflow/core/kernels/nth_element_op.cc create mode 100644 tensorflow/core/kernels/nth_element_op.h create mode 100644 tensorflow/python/kernel_tests/nth_element_op_test.py diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py index 7007e26bac..3bf795d19a 100644 --- a/tensorflow/contrib/nn/__init__.py +++ b/tensorflow/contrib/nn/__init__.py @@ -18,6 +18,7 @@ @@deprecated_flipped_softmax_cross_entropy_with_logits @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits @@deprecated_flipped_sigmoid_cross_entropy_with_logits +@@nth_element @@rank_sampled_softmax_loss @@scaled_softplus """ @@ -31,6 +32,7 @@ from tensorflow.contrib.nn.python.ops.alpha_dropout import * from tensorflow.contrib.nn.python.ops.cross_entropy import * from tensorflow.contrib.nn.python.ops.sampling_ops import * from tensorflow.contrib.nn.python.ops.scaled_softplus import * +from tensorflow.python.ops.nn_ops import nth_element # pylint: enable=unused-import,wildcard-import from tensorflow.python.util.all_util import remove_undocumented diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index b883be5d02..aba11c4c29 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2994,6 +2994,7 @@ cc_library( ":in_topk_op", ":l2loss_op", ":lrn_op", + ":nth_element_op", ":relu_op", ":softmax_op", ":softplus_op", @@ -3080,6 +3081,12 @@ tf_kernel_library( deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]), ) +tf_kernel_library( + name = "nth_element_op", + prefix = "nth_element_op", + deps = NN_DEPS, +) + tf_kernel_library( name = "xent_op", prefix = "xent_op", diff --git a/tensorflow/core/kernels/nth_element_op.cc b/tensorflow/core/kernels/nth_element_op.cc new file mode 100644 index 0000000000..da825e408c --- /dev/null +++ b/tensorflow/core/kernels/nth_element_op.cc @@ -0,0 +1,139 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/nn_ops.cc. +#include "tensorflow/core/kernels/nth_element_op.h" + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/work_sharder.h" +#include +#include +#include + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; + +template +class NthElementOp : public OpKernel { + public: + explicit NthElementOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("reverse", &reverse_)); + } + + void Compute(OpKernelContext* context) override { + // The second args is N, which must be a positive scalar. + const auto& n_in = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(n_in.shape()), + errors::InvalidArgument("N must be scalar, got shape ", + n_in.shape().DebugString())); + int n = n_in.scalar()(); + OP_REQUIRES(context, n >= 0, + errors::InvalidArgument("Need n >= 0, got ", n)); + + // The first args is input tensor, which must have 1 dimension at least. + const Tensor& input_in = context->input(0); + const int num_dims = input_in.dims(); + OP_REQUIRES(context, num_dims >= 1, + errors::InvalidArgument("Input must be >= 1-D, got shape ", + input_in.shape().DebugString())); + // The last dimension of input tensor must be greater than N. + OP_REQUIRES(context, input_in.dim_size(num_dims-1) > n, + errors::InvalidArgument("Input must have at least n+1 columns")); + + // std::nth_element only support the nth-smallest selection. + if (reverse_) { + n = input_in.dim_size(num_dims - 1) - n - 1; + } + + // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1]. + TensorShape out_shape; + for (int i = 0; i < num_dims-1; ++i) { + out_shape.AddDim(input_in.dim_size(i)); + } + Tensor* output_tensor = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, out_shape, &output_tensor)); + + functor::NthElementFunctor nthElementFunc; + nthElementFunc(context, input_in, *output_tensor, n, reverse_); + } + + private: + bool reverse_; +}; + +namespace functor { + +template +struct NthElementFunctor { + void operator() (OpKernelContext* context, + const Tensor& input_tensor, + Tensor& output_tensor, + int n, + bool reverse) { + const T* input = input_tensor.flat().data(); + T* output = output_tensor.flat().data(); + + // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1], + // then num_rows = d1*d2...dk-1, last_dim = dk. + const int num_rows = output_tensor.NumElements(); + const int last_dim = input_tensor.dim_size(input_tensor.dims()-1); + + // Allocate each row to different shard. + auto SubNthElement = [&, input, output, last_dim, n](int start, + int limit) { + // std::nth_element would rearrange the array, so we need a new buffer. + std::vector buf(last_dim); + + for (int b = start; b < limit; ++b) { + // Copy from one row of elements to buffer + const T* input_start = input + b * last_dim; + const T* input_end = input + (b+1) * last_dim; + std::copy(input_start, input_end, buf.begin()); + + std::nth_element(buf.begin(), buf.begin()+n, buf.end()); + // The element placed in the nth position is exactly the element that + // would occur in this position if the range was fully sorted. + output[b] = buf[n]; + } + }; + + auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); + // The average time complexity of partition-based nth_element (BFPRT) is O(n), + // althought the worst time complexity could be O(n^2). + // Here, 20 is a empirical factor of cost_per_unit. + Shard(worker_threads.num_threads, worker_threads.workers, num_rows, + 20 * last_dim, SubNthElement); + } +}; + +} // namespace functor + + +#define REGISTER_NTHOP(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("NthElement").Device(DEVICE_CPU).TypeConstraint("T"), \ + NthElementOp) + +TF_CALL_REAL_NUMBER_TYPES(REGISTER_NTHOP); +#undef REGISTER_NTHOP + +} // end namespace tensorflow + diff --git a/tensorflow/core/kernels/nth_element_op.h b/tensorflow/core/kernels/nth_element_op.h new file mode 100644 index 0000000000..11a6c996b0 --- /dev/null +++ b/tensorflow/core/kernels/nth_element_op.h @@ -0,0 +1,39 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_NTH_ELEMENT_OP_H_ +#define TENSORFLOW_NTH_ELEMENT_OP_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +namespace functor { + +template +struct NthElementFunctor { + void operator() (OpKernelContext* context, + const Tensor& input_tensor, + Tensor& output_tensor, + int n); +}; + +} // namespace functor + +} // namespace tensorflow + +#endif // TENSORFLOW_NTH_ELEMENT_OP_H_ diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 5efa55b496..1d26660a4b 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -2260,6 +2260,56 @@ indices: The indices of `values` within the last dimension of `input`. // -------------------------------------------------------------------------- +REGISTER_OP("NthElement") + .Input("input: T") + .Input("n: int32") + .Output("values: T") + .Attr("reverse: bool = false") + .Attr("T: realnumbertype") + .SetShapeFn([](InferenceContext* c) { + ShapeHandle input; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); + + // Get the n value from input tensor, and make sure which is a scalar. + DimensionHandle n_dim; + TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(1, &n_dim)); + + // The last dimension of input tensor must be greater than N. + DimensionHandle last_dim = c->Dim(input, -1); + if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) && + c->Value(last_dim) <= c->Value(n_dim)) { + return errors::InvalidArgument( + "Input must have last dimension > n = ", c->Value(n_dim), " but is ", + c->Value(last_dim)); + } + + // Reduce last_dim for output tensor + ShapeHandle s; + TF_RETURN_IF_ERROR(c->Subshape(input, 0, -1, &s)); + c->set_output(0, s); + return Status::OK(); + }) + .Doc(R"doc( +Finds values of the `n`-th order statistic for the last dmension. + +If the input is a vector (rank-1), finds the entries which is the nth-smallest +value in the vector and outputs their values as scalar tensor. + +For matrices (resp. higher rank input), computes the entries which is the +nth-smallest value in each row (resp. vector along the last dimension). Thus, + + values.shape = input.shape[:-1] + +input: 1-D or higher with last dimension at least `n+1`. +n: 0-D. Position of sorted vector to select along the last dimension (along + each row for matrices). Valid range of n is `[0, input.shape[:-1])` +reverse: When set to True, find the nth-largest value in the vector and vice + versa. +values: The `n`-th order statistic along each last dimensional slice. +)doc"); + +// -------------------------------------------------------------------------- + REGISTER_OP("FractionalMaxPool") .Input("value: T") .Output("output: T") diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc index 4628b725f8..94ecf4d5db 100644 --- a/tensorflow/core/ops/nn_ops_test.cc +++ b/tensorflow/core/ops/nn_ops_test.cc @@ -81,6 +81,30 @@ TEST(NNOpsTest, TopKV2_ShapeFn) { op, "[1,2,3,4];[]"); } +TEST(NNOpsTest, NthElement_ShapeFn) { + ShapeInferenceTestOp op("NthElement"); + op.input_tensors.resize(2); + + Tensor n_t; + op.input_tensors[1] = &n_t; + n_t = test::AsScalar(20); + + INFER_OK(op, "?;[]", "?"); + INFER_OK(op, "[21];[]", "[]"); + INFER_OK(op, "[2,?,?];[]", "[d0_0,d0_1]"); + INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]"); + + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]"); + INFER_ERROR("Input must have last dimension > n = 20 but is 1", op, + "[1];[]"); + INFER_ERROR("Input must have last dimension > n = 20 but is 20", op, + "[1,2,3,20];[]"); + n_t = test::AsScalar(-1); + INFER_ERROR( + "Dimension size, given by scalar input 1, must be non-negative but is -1", + op, "[1,2,3,4];[]"); +} + TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) { ShapeInferenceTestOp op("BatchNormWithGlobalNormalization"); diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 847c078971..127845e7d8 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -918,6 +918,21 @@ cuda_py_test( ], ) +cuda_py_test( + name = "nth_element_op_test", + size = "small", + srcs = ["nth_element_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:gradients", + "//tensorflow/python:nn_grad", + "//tensorflow/python:nn_ops", + ], +) + tf_py_test( name = "unique_op_test", size = "small", diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py new file mode 100644 index 0000000000..58cd46d2d5 --- /dev/null +++ b/tensorflow/python/kernel_tests/nth_element_op_test.py @@ -0,0 +1,174 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +import tensorflow.python.ops.nn_grad # pylint: disable=unused-import +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradients_impl +from tensorflow.python.platform import test + + +class NthElementTest(test.TestCase): + + def _validateNthElement(self, inputs, dtype, n, reverse, expected_values): + np_expected_values = np.array(expected_values) + with self.test_session(use_gpu=False) as sess: + inputs_op = ops.convert_to_tensor(inputs, dtype=dtype) + values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse) + values = sess.run(values_op) + + self.assertShapeEqual(np_expected_values, values_op) + self.assertAllClose(np_expected_values, values) + + def testExample1(self): + inputs = [2.2, 4.4, 1.1, 5.5, 3.3] + self._validateNthElement(inputs, dtypes.float32, 1, False, 2.2) + self._validateNthElement(inputs, dtypes.float32, 1, True, 4.4) + + def testExample2(self): + inputs = [[2.2, 4.4, 1.1], [5.5, 3.3, 6.6]] + self._validateNthElement(inputs, dtypes.float64, 2, False, [4.4, 6.6]) + self._validateNthElement(inputs, dtypes.float64, 2, True, [1.1, 3.3]) + + def testExample3(self): + inputs = [[[2, 4, 1], [5, -3, 6]], + [[7, 9, -8], [9, 0, 4]]] + self._validateNthElement(inputs, dtypes.int32, 0, False, + [[1, -3], [-8, 0]]) + self._validateNthElement(inputs, dtypes.int64, 0, True, + [[4, 6], [9, 9]]) + + def _testFloatLargeInput(self, input_shape): + inputs = np.random.random_sample(input_shape) + n = np.random.randint(input_shape[-1]) + sort_inputs = np.sort(inputs) + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.float32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.float64, n, True, expected_values) + + def _testIntLargeInput(self, input_shape): + inputs = np.random.randint(-1e3, 1e3, input_shape) + n = np.random.randint(input_shape[-1]) + sort_inputs = np.sort(inputs) + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.int32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.int64, n, True, expected_values) + + def _testLargeInput(self, input_shape): + self._testFloatLargeInput(input_shape) + self._testIntLargeInput(input_shape) + + def testLargeInput(self): + self._testLargeInput([1]) + self._testLargeInput([10]) + self._testLargeInput([5, 10]) + self._testLargeInput([50, 100]) + self._testLargeInput([50, 10000]) + self._testLargeInput([50, 10, 100]) + self._testLargeInput([50, 10, 10, 100]) + + def _testEnumerateN(self, input_shape): + inputs = np.random.random_sample(input_shape) + sort_inputs = np.sort(inputs) + for n in range(input_shape[-1]): + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.float32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.float64, n, True, expected_values) + + def testEnumerateN(self): + self._testEnumerateN([1]) + self._testEnumerateN([10]) + self._testEnumerateN([10, 10]) + self._testEnumerateN([10, 10, 10]) + self._testEnumerateN([10, 10, 10, 10]) + + def testInvalidInput(self): + with self.assertRaisesRegexp(ValueError, + "at least rank 1 but is rank 0"): + nn_ops.nth_element(5, 0) + + def testInvalidInputAtEval(self): + with self.test_session(use_gpu=False): + v = array_ops.placeholder(dtype=dtypes.float32) + with self.assertRaisesOpError("Input must be >= 1-D"): + nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0}) + + def testInvalidN(self): + with self.assertRaisesRegexp(ValueError, + "non-negative but is -1"): + nn_ops.nth_element([5], -1) + with self.assertRaisesRegexp(ValueError, + "scalar but has rank 1"): + nn_ops.nth_element([5, 6, 3], [1]) + + def testInvalidNAtEval(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.test_session(use_gpu=False): + n = array_ops.placeholder(dtypes.int32) + values = nn_ops.nth_element(inputs, n) + with self.assertRaisesOpError("Need n >= 0, got -7"): + values.eval(feed_dict={n: -7}) + + def testNTooLarge(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.assertRaisesRegexp(ValueError, + "must have last dimension > n = 2"): + nn_ops.nth_element(inputs, 2) + + def testNTooLargeAtEval(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.test_session(use_gpu=False): + n = array_ops.placeholder(dtypes.int32) + values = nn_ops.nth_element(inputs, n) + with self.assertRaisesOpError(r"Input must have at least n\+1 columns"): + values.eval(feed_dict={n: 2}) + + def testGradients(self): + with self.test_session(use_gpu=False) as sess: + inputs = array_ops.placeholder(dtypes.int32, shape=[3, 5]) + values = nn_ops.nth_element(inputs, 3) + grad = sess.run( + gradients_impl.gradients( + values, inputs, grad_ys=[[-1., 2., 5.]]), + feed_dict={inputs: [[2, -1, 1000, 3, 1000], + [1, 5, 2, 4, 3], + [2, 2, 2, 2, 2], + ]}) + self.assertAllClose(grad[0], [[0, 0, -0.5, 0, -0.5], + [0, 0, 0, 2, 0], + [1, 1, 1, 1, 1], + ]) + + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index af610d8fdb..c7c745142b 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -934,3 +934,32 @@ def _TopKGrad(op, grad, _): validate_indices=False), in_shape), array_ops.zeros( [], dtype=dtypes.int32)] + + +@ops.RegisterGradient("NthElement") +def _NthElementGrad(op, grad): + """Return the gradients for NthElement. + + Args: + op: The NthElementOp for which we need to generate gradients. + grad: Tensor. The gradients passed to the NthElementOp + + Returns: + A list of two tensors, the first being the gradient w.r.t. the input, + the second being the gradient w.r.t. the N (None). + """ + input = op.inputs[0] + output = op.outputs[0] + + # Compute the number of elements which equal to output in each reduction + # dimension. If there are multiple elements then the gradient will be + # divided between them. + indicators = math_ops.cast( + math_ops.equal(array_ops.expand_dims(output, -1), input), + grad.dtype) + + grad = array_ops.expand_dims(grad, -1) + num_selected = array_ops.expand_dims( + math_ops.reduce_sum(indicators, -1), -1) + + return [math_ops.div(indicators, num_selected) * grad, None] diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 8876591e53..8741b37c6f 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -2140,6 +2140,34 @@ def top_k(input, k=1, sorted=True, name=None): return gen_nn_ops._top_kv2(input, k=k, sorted=sorted, name=name) +def nth_element(input, n, reverse=False, name=None): + r"""Finds values of the `n`-th order statistic for the last dmension. + + If the input is a vector (rank-1), finds the entries which is the nth-smallest + value in the vector and outputs their values as scalar tensor. + + For matrices (resp. higher rank input), computes the entries which is the + nth-smallest value in each row (resp. vector along the last dimension). Thus, + + values.shape = input.shape[:-1] + + Args: + input: 1-D or higher `Tensor` with last dimension at least `n+1`. + n: A `Tensor` of type `int32`. + 0-D. Position of sorted vector to select along the last dimension (along + each row for matrices). Valid range of n is `[0, input.shape[:-1])` + reverse: An optional `bool`. Defaults to `False`. + When set to True, find the nth-largest value in the vector and vice + versa. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + The `n`-th order statistic along each last dimensional slice. + """ + return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name) + + def conv1d(value, filters, stride, padding, use_cudnn_on_gpu=None, data_format=None, name=None): -- GitLab From c27a2ecb57c6059235d4ee5e14697f95d1c21235 Mon Sep 17 00:00:00 2001 From: codrut3 Date: Wed, 18 Oct 2017 21:23:02 +0300 Subject: [PATCH 102/573] Allow num parameter in tf.linspace to be int64. (#13755) * Allow num parameter in tf.linspace to be int64. Currently tf.linspace is defined for num int32 or int64. However the kernel only allows int32, even though the op in core/ops/math_ops permits int64 too. I slightly changed the kernel to allow int64 too. I also added tests for RangeOp and LinSpaceOp. * Change variable names. --- tensorflow/core/kernels/BUILD | 18 +++ tensorflow/core/kernels/sequence_ops.cc | 48 +++--- tensorflow/core/kernels/sequence_ops_test.cc | 148 +++++++++++++++++++ 3 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 tensorflow/core/kernels/sequence_ops_test.cc diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index aba11c4c29..3a06189d72 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2636,6 +2636,24 @@ tf_kernel_library( deps = MATH_DEPS, ) +tf_cc_test( + name = "sequence_ops_test", + size = "small", + srcs = ["sequence_ops_test.cc"], + deps = [ + ":ops_testutil", + ":ops_util", + ":sequence_ops", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_cuda_cc_test( name = "cast_op_test", size = "small", diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc index c8ea923020..e2e3758d87 100644 --- a/tensorflow/core/kernels/sequence_ops.cc +++ b/tensorflow/core/kernels/sequence_ops.cc @@ -96,7 +96,7 @@ TF_CALL_double(REGISTER_SYCL_KERNEL); TF_CALL_int32(REGISTER_SYCL_KERNEL); TF_CALL_int64(REGISTER_SYCL_KERNEL); #undef REGISTER_SYCL_KERNEL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TF_CALL_float(REGISTER_CPU_KERNEL); TF_CALL_double(REGISTER_CPU_KERNEL); @@ -116,7 +116,7 @@ TF_CALL_int64(REGISTER_GPU_KERNEL); #undef REGISTER_CPU_KERNEL #undef REGISTER_GPU_KERNEL -template +template class LinSpaceOp : public OpKernel { public: explicit LinSpaceOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -136,7 +136,7 @@ class LinSpaceOp : public OpKernel { num_in.shape().DebugString())); const T start = start_in.scalar()(); const T stop = stop_in.scalar()(); - const int32 num = num_in.scalar()(); + const Tnum num = num_in.scalar()(); OP_REQUIRES(context, num > 0, errors::InvalidArgument("Requires num > 0: ", num)); Tensor* out = nullptr; @@ -147,34 +147,46 @@ class LinSpaceOp : public OpKernel { flat(0) = start; } else { const T step = (stop - start) / (num - 1); - for (int32 i = 0; i < num; ++i) flat(i) = start + step * i; + for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i; } } }; -#define REGISTER_KERNEL(DEV, T) \ - REGISTER_KERNEL_BUILDER(Name("LinSpace") \ - .Device(DEV) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("start") \ - .HostMemory("stop") \ - .HostMemory("num") \ - .HostMemory("output"), \ - LinSpaceOp); -#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL(DEVICE_CPU, T) +#define REGISTER_KERNEL(DEV, T, Tidx) \ + REGISTER_KERNEL_BUILDER(Name("LinSpace") \ + .Device(DEV) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("start") \ + .HostMemory("stop") \ + .HostMemory("num") \ + .HostMemory("output"), \ + LinSpaceOp); + +#define REGISTER_KERNEL_ALL_NUMS(dev, T) \ + REGISTER_KERNEL(dev, T, int32); \ + REGISTER_KERNEL(dev, T, int64) + +#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_CPU, T) TF_CALL_float(REGISTER_CPU_KERNEL); TF_CALL_double(REGISTER_CPU_KERNEL); // NOTE(touts): We register the op on GPU but it still runs on CPU // because its inputs and outputs are tagged as HostMemory. -#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL(DEVICE_GPU, T) +#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_GPU, T) TF_CALL_float(REGISTER_GPU_KERNEL); TF_CALL_double(REGISTER_GPU_KERNEL); +#undef REGISTER_GPU_KERNEL #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL(DEVICE_SYCL, T) +#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_SYCL, T) TF_CALL_float(REGISTER_SYCL_KERNEL); TF_CALL_double(REGISTER_SYCL_KERNEL); -#endif // TENSORFLOW_USE_SYCL +#undef REGISTER_SYCL_KERNEL +#endif // TENSORFLOW_USE_SYCL + +#undef REGISTER_CPU_KERNEL +#undef REGISTER_KERNEL_ALL_NUMS +#undef REGISTER_KERNEL + } // namespace tensorflow diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc new file mode 100644 index 0000000000..5f0e0a69a8 --- /dev/null +++ b/tensorflow/core/kernels/sequence_ops_test.cc @@ -0,0 +1,148 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/ops_testutil.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +class RangeOpTest : public OpsTestBase { + protected: + void MakeOp(DataType input_type) { + TF_ASSERT_OK(NodeDefBuilder("myop", "Range") + .Input(FakeInput(input_type)) + .Input(FakeInput(input_type)) + .Input(FakeInput(input_type)) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + } +}; + +class LinSpaceOpTest : public OpsTestBase { + protected: + void MakeOp(DataType input_type, DataType index_type) { + TF_ASSERT_OK(NodeDefBuilder("myop", "LinSpace") + .Input(FakeInput(input_type)) + .Input(FakeInput(input_type)) + .Input(FakeInput(index_type)) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + } +}; + +TEST_F(RangeOpTest, Simple_D32) { + MakeOp(DT_INT32); + + // Feed and run + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {10}); + AddInputFromArray(TensorShape({}), {2}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_INT32, TensorShape({5})); + test::FillValues(&expected, {0, 2, 4, 6, 8}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(RangeOpTest, Simple_Float) { + MakeOp(DT_FLOAT); + + // Feed and run + AddInputFromArray(TensorShape({}), {0.5}); + AddInputFromArray(TensorShape({}), {2}); + AddInputFromArray(TensorShape({}), {0.3}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_FLOAT, TensorShape({5})); + test::FillValues(&expected, {0.5, 0.8, 1.1, 1.4, 1.7}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(RangeOpTest, Large_Double) { + MakeOp(DT_DOUBLE); + + // Feed and run + AddInputFromArray(TensorShape({}), {0.0}); + AddInputFromArray(TensorShape({}), {10000}); + AddInputFromArray(TensorShape({}), {0.5}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_DOUBLE, TensorShape({20000})); + std::vector result; + for (int32 i = 0; i < 20000; ++i) result.push_back(i * 0.5); + test::FillValues(&expected, gtl::ArraySlice(result)); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(LinSpaceOpTest, Simple_D32) { + MakeOp(DT_FLOAT, DT_INT32); + + // Feed and run + AddInputFromArray(TensorShape({}), {3.0}); + AddInputFromArray(TensorShape({}), {7.0}); + AddInputFromArray(TensorShape({}), {3}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_FLOAT, TensorShape({3})); + test::FillValues(&expected, {3.0, 5.0, 7.0}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(LinSpaceOpTest, Single_D64) { + MakeOp(DT_FLOAT, DT_INT64); + + // Feed and run + AddInputFromArray(TensorShape({}), {9.0}); + AddInputFromArray(TensorShape({}), {100.0}); + AddInputFromArray(TensorShape({}), {1}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_FLOAT, TensorShape({1})); + test::FillValues(&expected, {9.0}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(LinSpaceOpTest, Simple_Double) { + MakeOp(DT_DOUBLE, DT_INT32); + + // Feed and run + AddInputFromArray(TensorShape({}), {5.0}); + AddInputFromArray(TensorShape({}), {6.0}); + AddInputFromArray(TensorShape({}), {6}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_DOUBLE, TensorShape({6})); + test::FillValues(&expected, {5.0, 5.2, 5.4, 5.6, 5.8, 6.0}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +} // namespace +} // namespace tensorflow -- GitLab From 21f68a85f30ddea7e4a66dbe70d18069a2a1d0a1 Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Wed, 18 Oct 2017 11:27:38 -0700 Subject: [PATCH 103/573] Remove global step read dependency from model_fn. Estimator behavior still will be deterministic since the step checking logic in session_run_hooks was changed as follows: * assume stale step * before using the step, check for the current value by session.run PiperOrigin-RevId: 172629797 --- .../contrib/learn/python/learn/estimators/estimator.py | 5 ++--- tensorflow/python/estimator/estimator.py | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py index 8bb1c83a45..788d2d0b1a 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py @@ -981,9 +981,8 @@ class BaseEstimator( global_step = training_util.create_global_step(g) features, labels = input_fn() self._check_inputs(features, labels) - global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access - with ops.control_dependencies([global_step_read_tensor]): - model_fn_ops = self._get_train_ops(features, labels) + training_util._get_or_create_global_step_read() # pylint: disable=protected-access + model_fn_ops = self._get_train_ops(features, labels) ops.add_to_collection(ops.GraphKeys.LOSSES, model_fn_ops.loss) all_hooks.extend(hooks) all_hooks.extend([ diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 00a57f11dc..2a4d77b1a6 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -707,12 +707,11 @@ class Estimator(object): with ops.Graph().as_default() as g, g.device(self._device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) - global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access + training_util._get_or_create_global_step_read() # pylint: disable=protected-access features, labels = self._get_features_and_labels_from_input_fn( input_fn, model_fn_lib.ModeKeys.TRAIN) - with ops.control_dependencies([global_step_read_tensor]): - estimator_spec = self._call_model_fn( - features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) + estimator_spec = self._call_model_fn( + features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) # Check if the user created a loss summary, and add one if they didn't. # We assume here that the summary is called 'loss'. If it is not, we will # make another one with the name 'loss' to ensure it shows up in the right -- GitLab From 1a325f1330cd6e1204d47f1fcf64caf402788477 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Wed, 18 Oct 2017 11:35:12 -0700 Subject: [PATCH 104/573] More changs to avoid flakes in random_shuffle_queue_test PiperOrigin-RevId: 172630989 --- .../kernel_tests/random_shuffle_queue_test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py index 1b84af6823..c4e16ff628 100644 --- a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py +++ b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py @@ -654,7 +654,8 @@ class RandomShuffleQueueTest(test.TestCase): def testBlockingDequeueFromClosedQueue(self): with self.test_session() as sess: - q = data_flow_ops.RandomShuffleQueue(10, 2, dtypes_lib.float32) + min_size = 2 + q = data_flow_ops.RandomShuffleQueue(10, min_size, dtypes_lib.float32) elems = [10.0, 20.0, 30.0, 40.0] enqueue_op = q.enqueue_many((elems,)) close_op = q.close() @@ -664,20 +665,24 @@ class RandomShuffleQueueTest(test.TestCase): results = [] - def dequeue(): - for _ in elems: - results.append(sess.run(dequeued_t)) + # Manually dequeue until we hit min_size. + results.append(sess.run(dequeued_t)) + results.append(sess.run(dequeued_t)) + + def blocking_dequeue(): + results.append(sess.run(dequeued_t)) + results.append(sess.run(dequeued_t)) + self.assertItemsEqual(elems, results) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): sess.run(dequeued_t) - dequeue_thread = self.checkedThread(target=dequeue) + dequeue_thread = self.checkedThread(target=blocking_dequeue) dequeue_thread.start() - # The close_op should run after the dequeue_thread has blocked. - # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) + # The dequeue thread blocked when it hit the min_size requirement. self.assertEqual(len(results), 2) close_op.run() -- GitLab From 6a725f6d0974dc71fe4ac311fc8dd16db4257452 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 11:57:04 -0700 Subject: [PATCH 105/573] Add expected keys to predictor exception if unexpected key detected. PiperOrigin-RevId: 172634275 --- tensorflow/contrib/predictor/predictor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/predictor/predictor.py b/tensorflow/contrib/predictor/predictor.py index dbc0028259..28fa815684 100644 --- a/tensorflow/contrib/predictor/predictor.py +++ b/tensorflow/contrib/predictor/predictor.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - """Abstract base class for all predictors.""" from __future__ import absolute_import @@ -66,8 +65,9 @@ class Predictor(object): expected_keys = set(self.feed_tensors.keys()) unexpected_keys = input_keys - expected_keys if unexpected_keys: - raise ValueError('Got unexpected keys in input_dict: {}'.format( - unexpected_keys)) + raise ValueError( + 'Got unexpected keys in input_dict: {}\nexpected: {}'.format( + unexpected_keys, expected_keys)) feed_dict = {} for key in self.feed_tensors.keys(): -- GitLab From f5d3bf42b892ecfbde2ce9eb45f00b76473c824a Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Wed, 18 Oct 2017 11:58:01 -0700 Subject: [PATCH 106/573] Add TF_GraphGetOpDef() to C API and use in Operation.op_def() Note that this creates a small change in behavior with the C API enabled, since previously not all Python Operations had an OpDef (op_def() returns None). With the C API enabled, op_def() always returns an OpDef. PiperOrigin-RevId: 172634411 --- tensorflow/c/c_api.cc | 11 +++++++++ tensorflow/c/c_api.h | 7 ++++++ tensorflow/c/c_api_function_test.cc | 21 +++++++++++++++++ tensorflow/c/c_api_test.cc | 31 +++++++++++++++++++++++++ tensorflow/python/framework/ops.py | 15 +++++++++++- tensorflow/python/framework/ops_test.py | 15 ++++++++++++ 6 files changed, 99 insertions(+), 1 deletion(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 79fbd8c90c..cd98393e0a 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -1799,6 +1799,17 @@ void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def, status->status = MessageToBuffer(def, output_graph_def); } +void TF_GraphGetOpDef(TF_Graph* graph, const char* op_name, + TF_Buffer* output_op_def, TF_Status* status) { + const OpDef* op_def; + { + mutex_lock l(graph->mu); + status->status = graph->graph.op_registry()->LookUpOpDef(op_name, &op_def); + if (!status->status.ok()) return; + } + status->status = MessageToBuffer(*op_def, output_op_def); +} + TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions() { return new TF_ImportGraphDefOptions; } diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index 68a758498d..0c6bb53d01 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -864,6 +864,13 @@ TF_CAPI_EXPORT extern void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def, TF_Status* status); +// Returns the serialized OpDef proto with name `op_name`, or a bad status if no +// such op exists. This can return OpDefs of functions copied into the graph. +TF_CAPI_EXPORT extern void TF_GraphGetOpDef(TF_Graph* graph, + const char* op_name, + TF_Buffer* output_op_def, + TF_Status* status); + // TF_ImportGraphDefOptions holds options that can be passed to // TF_GraphImportGraphDef. typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions; diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc index 4db9a90fdc..d5580b6589 100644 --- a/tensorflow/c/c_api_function_test.cc +++ b/tensorflow/c/c_api_function_test.cc @@ -1465,5 +1465,26 @@ TEST_F(CApiFunctionTest, AppendHash) { ASSERT_EQ(string("func_name_base_qaJ8jA8UmGY"), fdef.signature().name()); } +TEST_F(CApiFunctionTest, GetOpDef) { + DefineFunction(func_name_, &func_); + TF_GraphCopyFunction(host_graph_, func_, nullptr, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Test we can retrieve function OpDef from graph + TF_Buffer* buffer = TF_NewBuffer(); + TF_GraphGetOpDef(host_graph_, func_name_, buffer, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Sanity check returned OpDef + string data(static_cast(buffer->data), buffer->length); + OpDef op_def; + op_def.ParseFromString(data); + EXPECT_EQ(op_def.name(), func_name_); + EXPECT_EQ(op_def.input_arg_size(), 1); + EXPECT_EQ(op_def.output_arg_size(), 1); + + TF_DeleteBuffer(buffer); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc index c442029009..d220bc5e95 100644 --- a/tensorflow/c/c_api_test.cc +++ b/tensorflow/c/c_api_test.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/graph/tensor_id.h" #include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -50,6 +51,11 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst); namespace { +static void ExpectHasSubstr(StringPiece s, StringPiece expected) { + EXPECT_TRUE(StringPiece(s).contains(expected)) + << "'" << s << "' does not contain '" << expected << "'"; +} + TEST(CAPI, Version) { EXPECT_STRNE("", TF_Version()); } TEST(CAPI, Status) { @@ -837,6 +843,31 @@ TEST(CAPI, ShapeInferenceError) { TF_DeleteStatus(status); } +TEST(CAPI, GetOpDef) { + TF_Status* status = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + TF_Buffer* buffer = TF_NewBuffer(); + + TF_GraphGetOpDef(graph, "Add", buffer, status); + ASSERT_EQ(TF_OK, TF_GetCode(status)); + const OpDef* expected_op_def; + TF_ASSERT_OK(OpRegistry::Global()->LookUpOpDef("Add", &expected_op_def)); + string expected_serialized; + expected_op_def->SerializeToString(&expected_serialized); + string actual_string(reinterpret_cast(buffer->data), + buffer->length); + EXPECT_EQ(expected_serialized, actual_string); + + TF_GraphGetOpDef(graph, "MyFakeOp", buffer, status); + EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status)); + ExpectHasSubstr(TF_Message(status), + "Op type not registered 'MyFakeOp' in binary"); + + TF_DeleteBuffer(buffer); + TF_DeleteGraph(graph); + TF_DeleteStatus(status); +} + void StringVectorToArrays(const std::vector& v, std::unique_ptr* ptrs, std::unique_ptr* lens) { diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 3ac8a0cb6a..75750ecd5a 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -33,6 +33,7 @@ from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import function_pb2 from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 +from tensorflow.core.framework import op_def_pb2 from tensorflow.core.framework import versions_pb2 from tensorflow.python import pywrap_tensorflow as c_api from tensorflow.python.eager import context @@ -1985,7 +1986,19 @@ class Operation(object): protocol buffer. """ # pylint: enable=line-too-long - return self._op_def + if self._c_op: + with errors.raise_exception_on_not_ok_status() as status: + with c_api_util.tf_buffer() as buf: + # pylint: disable=protected-access + c_api.TF_GraphGetOpDef(self._graph._c_graph, + compat.as_bytes(self.type), buf, status) + # pylint: enable=protected-access + data = c_api.TF_GetBuffer(buf) + op_def = op_def_pb2.OpDef() + op_def.ParseFromString(compat.as_bytes(data)) + return op_def + else: + return self._op_def @property def traceback(self): diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index f20c808cde..59c0288457 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -504,6 +504,21 @@ class OperationTest(test_util.TensorFlowTestCase): r"num of inputs: 0\) does not have input 1"): x.op._update_input(1, x) # pylint: disable=protected-access + def testOpDef(self): + x = constant_op.constant(0) + y = constant_op.constant(1) + z = x + y + + # Pure Python mode doesn't create OpDefs for constants + if ops._USE_C_API: + self.assertEqual(x.op.op_def.name, "Const") + self.assertEqual(len(x.op.op_def.input_arg), 0) + self.assertEqual(len(x.op.op_def.output_arg), 1) + + self.assertEqual(z.op.op_def.name, "Add") + self.assertEqual(len(z.op.op_def.input_arg), 2) + self.assertEqual(len(z.op.op_def.output_arg), 1) + @test_util.with_c_api class CreateOpTest(test_util.TensorFlowTestCase): -- GitLab From f5ea388e48a38b935ebd36442f756c8974b7ce3f Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Wed, 18 Oct 2017 11:58:24 -0700 Subject: [PATCH 107/573] Implement ZlibInputStream::Tell() by keeping track of the number of bytes consumed by the reader. PiperOrigin-RevId: 172634455 --- tensorflow/core/lib/io/zlib_buffers_test.cc | 172 +++++++++++++++++--- tensorflow/core/lib/io/zlib_inputstream.cc | 8 +- tensorflow/core/lib/io/zlib_inputstream.h | 3 + 3 files changed, 156 insertions(+), 27 deletions(-) diff --git a/tensorflow/core/lib/io/zlib_buffers_test.cc b/tensorflow/core/lib/io/zlib_buffers_test.cc index 66ee68a916..156c712db8 100644 --- a/tensorflow/core/lib/io/zlib_buffers_test.cc +++ b/tensorflow/core/lib/io/zlib_buffers_test.cc @@ -68,25 +68,25 @@ void TestAllCombinations(CompressionOptions input_options, for (auto input_buf_size : InputBufferSizes()) { for (auto output_buf_size : OutputBufferSizes()) { std::unique_ptr file_writer; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); string result; ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); - TF_CHECK_OK(out.Append(StringPiece(data))); - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, input_options); - TF_EXPECT_OK(in.ReadNBytes(data.size(), &result)); + TF_ASSERT_OK(in.ReadNBytes(data.size(), &result)); EXPECT_EQ(result, data); } } @@ -118,24 +118,24 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size, string actual_result; string expected_result; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); for (int i = 0; i < num_writes; i++) { - TF_CHECK_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Append(StringPiece(data))); if (with_flush) { - TF_CHECK_OK(out.Flush()); + TF_ASSERT_OK(out.Flush()); } strings::StrAppend(&expected_result, data); } - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, @@ -143,7 +143,7 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size, for (int i = 0; i < num_writes; i++) { string decompressed_output; - TF_EXPECT_OK(in.ReadNBytes(data.size(), &decompressed_output)); + TF_ASSERT_OK(in.ReadNBytes(data.size(), &decompressed_output)); strings::StrAppend(&actual_result, decompressed_output); } @@ -170,19 +170,19 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) { string data = GenTestString(10); std::unique_ptr file_writer; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); string result; ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); - TF_CHECK_OK(out.Append(StringPiece(data))); - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, @@ -192,5 +192,129 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) { CHECK(read_status.error_message().find("inflate() failed") != string::npos); } +void WriteCompressedFile(Env* env, const string& fname, int input_buf_size, + int output_buf_size, + const CompressionOptions& output_options, + const string& data) { + std::unique_ptr file_writer; + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); + + ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, + output_options); + TF_ASSERT_OK(out.Init()); + + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); +} + +void TestTell(CompressionOptions input_options, + CompressionOptions output_options) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/zlib_buffers_test"; + for (auto file_size : NumCopies()) { + string data = GenTestString(file_size); + for (auto input_buf_size : InputBufferSizes()) { + for (auto output_buf_size : OutputBufferSizes()) { + // Write the compressed file. + WriteCompressedFile(env, fname, input_buf_size, output_buf_size, + output_options, data); + + // Boiler-plate to set up ZlibInputStream. + std::unique_ptr file_reader; + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); + std::unique_ptr input_stream( + new RandomAccessInputStream(file_reader.get())); + ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, + input_options); + + string first_half(data, 0, data.size() / 2); + string bytes_read; + + // Read the first half of the uncompressed file and expect that Tell() + // returns half the uncompressed length of the file. + TF_ASSERT_OK(in.ReadNBytes(first_half.size(), &bytes_read)); + EXPECT_EQ(in.Tell(), first_half.size()); + EXPECT_EQ(bytes_read, first_half); + + // Read the remaining half of the uncompressed file and expect that + // Tell() points past the end of file. + string second_half; + TF_ASSERT_OK( + in.ReadNBytes(data.size() - first_half.size(), &second_half)); + EXPECT_EQ(in.Tell(), data.size()); + bytes_read.append(second_half); + + // Expect that the file is correctly read. + EXPECT_EQ(bytes_read, data); + } + } + } +} + +void TestSkipNBytes(CompressionOptions input_options, + CompressionOptions output_options) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/zlib_buffers_test"; + for (auto file_size : NumCopies()) { + string data = GenTestString(file_size); + for (auto input_buf_size : InputBufferSizes()) { + for (auto output_buf_size : OutputBufferSizes()) { + // Write the compressed file. + WriteCompressedFile(env, fname, input_buf_size, output_buf_size, + output_options, data); + + // Boiler-plate to set up ZlibInputStream. + std::unique_ptr file_reader; + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); + std::unique_ptr input_stream( + new RandomAccessInputStream(file_reader.get())); + ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, + input_options); + + size_t data_half_size = data.size() / 2; + string second_half(data, data_half_size, data.size() - data_half_size); + + // Skip past the first half of the file and expect Tell() returns + // correctly. + TF_ASSERT_OK(in.SkipNBytes(data_half_size)); + EXPECT_EQ(in.Tell(), data_half_size); + + // Expect that second half is read correctly and Tell() returns past + // end of file after reading complete file. + string bytes_read; + TF_ASSERT_OK(in.ReadNBytes(second_half.size(), &bytes_read)); + EXPECT_EQ(bytes_read, second_half); + EXPECT_EQ(in.Tell(), data.size()); + } + } + } +} + +TEST(ZlibInputStream, TellDefaultOptions) { + TestTell(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT()); +} + +TEST(ZlibInputStream, TellRawDeflate) { + TestTell(CompressionOptions::RAW(), CompressionOptions::RAW()); +} + +TEST(ZlibInputStream, TellGzip) { + TestTell(CompressionOptions::GZIP(), CompressionOptions::GZIP()); +} + +TEST(ZlibInputStream, SkipNBytesDefaultOptions) { + TestSkipNBytes(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT()); +} + +TEST(ZlibInputStream, SkipNBytesRawDeflate) { + TestSkipNBytes(CompressionOptions::RAW(), CompressionOptions::RAW()); +} + +TEST(ZlibInputStream, SkipNBytesGzip) { + TestSkipNBytes(CompressionOptions::GZIP(), CompressionOptions::GZIP()); +} + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index 4999d5cc90..984fbc2810 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -32,7 +32,8 @@ ZlibInputStream::ZlibInputStream( z_stream_input_(new Bytef[input_buffer_capacity_]), z_stream_output_(new Bytef[output_buffer_capacity_]), zlib_options_(zlib_options), - z_stream_(new z_stream) { + z_stream_(new z_stream), + bytes_read_(0) { InitZlibBuffer(); } @@ -45,6 +46,7 @@ ZlibInputStream::~ZlibInputStream() { Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); InitZlibBuffer(); + bytes_read_ = 0; return Status::OK(); } @@ -127,6 +129,7 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read, result->append(next_unread_byte_, can_read_bytes); next_unread_byte_ += can_read_bytes; } + bytes_read_ += can_read_bytes; return can_read_bytes; } @@ -170,8 +173,7 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) { return Status::OK(); } -// TODO(srbs): Implement this. -int64 ZlibInputStream::Tell() const { return -1; } +int64 ZlibInputStream::Tell() const { return bytes_read_; } Status ZlibInputStream::Inflate() { int error = inflate(z_stream_.get(), zlib_options_.flush_mode); diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 8faa7dcb8f..9c7e14441c 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -132,6 +132,9 @@ class ZlibInputStream : public InputStreamInterface { // Returns the size of [next_unread_byte_, z_stream_->next_out) size_t NumUnreadBytes() const; + // Number of *uncompressed* bytes that have been read from this stream. + int64 bytes_read_; + TF_DISALLOW_COPY_AND_ASSIGN(ZlibInputStream); }; -- GitLab From d19ec7126735ca98a632ebd69ad64973fd454e6e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 18 Oct 2017 12:10:56 -0700 Subject: [PATCH 108/573] Update boringssl to a0fb951d2a26a8ee746b52f3ba81ab011a0af778 (#13798) * Update boringssl to a0fb951d2a26a8ee746b52f3ba81ab011a0af778 This fix update boringssl to a0fb951d2a26a8ee746b52f3ba81ab011a0af778, which contains bug fix related to 13733. See https://github.com/tensorflow/tensorflow/pull/13734#issuecomment-337440239 for update details. Signed-off-by: Yong Tang * Remove unneeded patch part for boringssl, as 13733 has been fixed. Signed-off-by: Yong Tang * Remove s390x patch for boringssl Per discussion: https://github.com/tensorflow/tensorflow/pull/13798#discussion_r145459255 Signed-off-by: Yong Tang --- tensorflow/workspace.bzl | 10 ++++---- .../boringssl/add_boringssl_s390x.patch | 23 ------------------- 2 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 third_party/boringssl/add_boringssl_s390x.patch diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index ea50d0f296..0997fffc8a 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -590,15 +590,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""): actual = "@jsoncpp_git//:jsoncpp", ) - patched_http_archive( + native.http_archive( name = "boringssl", urls = [ - "https://github.com/google/boringssl/archive/72cfd9f49ec5fbc2db368b76398c196dafe6a4bc.tar.gz", + "https://github.com/google/boringssl/archive/a0fb951d2a26a8ee746b52f3ba81ab011a0af778.tar.gz", ], - sha256 = "5e6f7b72c74adeb902581271925ddb979e77b96327abd76604ce894d80680e51", - strip_prefix = "boringssl-72cfd9f49ec5fbc2db368b76398c196dafe6a4bc", - # Add patch to boringssl code to support s390x - patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")), + sha256 = "524ba98a56300149696481b4cb9ddebd0c7b7ac9b9f6edee81da2d2d7e5d2bb3", + strip_prefix = "boringssl-a0fb951d2a26a8ee746b52f3ba81ab011a0af778", ) native.new_http_archive( diff --git a/third_party/boringssl/add_boringssl_s390x.patch b/third_party/boringssl/add_boringssl_s390x.patch deleted file mode 100644 index b684dc6df7..0000000000 --- a/third_party/boringssl/add_boringssl_s390x.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff -ur a/BUILD b/BUILD ---- a/BUILD 2017-10-10 15:50:34.000000000 +0000 -+++ b/BUILD 2017-10-15 21:19:02.057606476 +0000 -@@ -63,6 +63,7 @@ - "-Wwrite-strings", - "-Wshadow", - "-fno-common", -+ "-Wno-uninitialized", - - # Modern build environments should be able to set this to use atomic - # operations for reference counting rather than locks. However, it's -diff -ur a/src/include/openssl/base.h b/src/include/openssl/base.h ---- a/src/include/openssl/base.h 2017-10-10 15:50:34.000000000 +0000 -+++ b/src/include/openssl/base.h 2017-10-15 19:49:38.182154627 +0000 -@@ -106,6 +106,8 @@ - #define OPENSSL_PNACL - #elif defined(__myriad2__) - #define OPENSSL_32_BIT -+#elif defined(__s390x__) -+#define OPENSSL_64_BIT - #else - // Note BoringSSL only supports standard 32-bit and 64-bit two's-complement, - // little-endian architectures. Functions will not produce the correct answer -- GitLab From ef060d923acef07cd3a4b1134218abc84fcb3a7b Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Wed, 18 Oct 2017 12:06:25 -0700 Subject: [PATCH 109/573] Upgrade tensorflow pip dependency version to 3.4.0+ PiperOrigin-RevId: 172635727 --- tensorflow/tools/pip_package/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index a7a0706d0b..c05d39e942 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -35,7 +35,7 @@ REQUIRED_PACKAGES = [ 'enum34 >= 1.1.6', 'numpy >= 1.12.1', 'six >= 1.10.0', - 'protobuf >= 3.3.0', + 'protobuf >= 3.4.0', 'tensorflow-tensorboard >= 0.1.0, < 0.2.0', ] -- GitLab From f1603b7893f922dfe64244c6bae9b93d7d594437 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Wed, 18 Oct 2017 12:16:38 -0700 Subject: [PATCH 110/573] [XLA] Deterministically dump an executable. Previously, dumping a executable is nondeterministic as a map in protobuf is serialized in random order. This CL enables "Deterministic dump" mode of protobuf, which sorts the map first before dumping them. This is helpful in comparing if two dumps are the same in XLA determinism test. PiperOrigin-RevId: 172637100 --- tensorflow/compiler/xla/BUILD | 1 + tensorflow/compiler/xla/service/BUILD | 2 ++ tensorflow/compiler/xla/service/executable.cc | 8 +++++++- tensorflow/compiler/xla/util.h | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index be87506d3c..e51bbffcd0 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -171,6 +171,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":status", + ":status_macros", ":types", ":xla_data_proto", "//tensorflow/core:lib", diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index d1335e20e0..fed7bd01f6 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -581,12 +581,14 @@ cc_library( ":shaped_buffer", ":versioned_computation_handle", "//tensorflow/compiler/xla:executable_run_options", + "//tensorflow/compiler/xla:status", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/legacy_flags:debug_options_flags", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/stream_executor", ], diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index 62b8fa6a2b..9c96d9eb30 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -17,7 +17,9 @@ limitations under the License. #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h" #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h" +#include "tensorflow/compiler/xla/status.h" #include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/env.h" @@ -82,7 +84,11 @@ Status Executable::DumpSessionModule() { } filename = SanitizeFileName(std::move(filename)); string file_path = tensorflow::io::JoinPath(directory_path, filename); - return tensorflow::WriteBinaryProto(env, file_path, session_module); + string result; + TF_RET_CHECK( + tensorflow::SerializeToStringDeterministic(session_module, &result)); + return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path, + result); } } // namespace xla diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h index f6c0bd1563..f58f57b443 100644 --- a/tensorflow/compiler/xla/util.h +++ b/tensorflow/compiler/xla/util.h @@ -24,6 +24,7 @@ limitations under the License. #include #include "tensorflow/compiler/xla/status.h" +#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/core/status.h" -- GitLab From 192f1c24ec6692342391c03bb620f5de1af9de3b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 12:22:04 -0700 Subject: [PATCH 111/573] Fixed work size computation in Split and SplitV ops to avoid integer overflow. PiperOrigin-RevId: 172637818 --- tensorflow/core/kernels/split_op.cc | 8 ++++---- tensorflow/core/kernels/split_v_op.cc | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc index 4d2100c59c..58e1a73be6 100644 --- a/tensorflow/core/kernels/split_op.cc +++ b/tensorflow/core/kernels/split_op.cc @@ -167,11 +167,11 @@ class SplitOpCPU : public SplitOpBase { const auto num_threads = context->device()->tensorflow_cpu_worker_threads()->num_threads; // TODO(jewillco): Tune heuristic further. + const auto input_element_count = input_shape.num_elements(); const bool use_parallelism_between_outputs = (num_split >= 4 && - input_shape.num_elements() >= - std::max(num_threads, num_split) * 4096 && - input_shape.num_elements() < num_split * 180 * 1024); + input_element_count >= std::max(num_threads, num_split) * 4096 && + input_element_count < num_split * 180 * 1024); auto range_output_func = [&indices, context, &output_shape, prefix_dim_size, split_dim_output_size, suffix_dim_size, &sizes, @@ -209,7 +209,7 @@ class SplitOpCPU : public SplitOpBase { // Run in parallel, disabling parallelism in functor. Shard(num_split, context->device()->tensorflow_cpu_worker_threads()->workers, - num_split, kint64max, range_output_func); + num_split, input_element_count / num_split, range_output_func); } else { // Run sequentially, but allow internal parallelism in functor. range_output_func(0, num_split); diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc index e2dd66da1e..3316e5fcc9 100644 --- a/tensorflow/core/kernels/split_v_op.cc +++ b/tensorflow/core/kernels/split_v_op.cc @@ -225,11 +225,11 @@ class SplitVOpCPU : public SplitVOpBase { const auto num_threads = context->device()->tensorflow_cpu_worker_threads()->num_threads; // TODO(jewillco): Tune heuristic further. + const auto input_element_count = input_shape.num_elements(); const bool use_parallelism_between_outputs = (num_split >= 4 && - input_shape.num_elements() >= - std::max(num_threads, num_split) * 4096 && - input_shape.num_elements() < num_split * 180 * 1024); + input_element_count >= std::max(num_threads, num_split) * 4096 && + input_element_count < num_split * 180 * 1024); auto range_output_func = [&indices, context, &input_shape, prefix_dim_size, split_dim, &split_sizes_vec, &split_start_points, @@ -267,7 +267,7 @@ class SplitVOpCPU : public SplitVOpBase { // Run in parallel, disabling parallelism in functor. Shard(num_split, context->device()->tensorflow_cpu_worker_threads()->workers, - num_split, kint64max, range_output_func); + num_split, input_element_count / num_split, range_output_func); } else { // Run sequentially, but allow internal parallelism in functor. range_output_func(0, num_split); -- GitLab From 7321905ff14c47211c95e430625f8b29986c1df1 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 18 Oct 2017 12:48:13 -0700 Subject: [PATCH 112/573] Add missing `uint16` type registration for ops `CropAndResize`/`CropAndResizeGradBoxes` (#13812) * Add missing `uint16` type registration for ops `CropAndResize`/`CropAndResizeGradBoxes` This fix adds missing `uint16` type registration in `image_ops.cc` for `CropAndResize` and `CropAndResizeGradBoxes`. The kernel of `uint16` is available for `CropAndResize` and `CropAndResizeGradBoxes` though it is missing in `image_ops.cc`. This fix addresses this issue. Signed-off-by: Yong Tang * Add incomplete test cases for `CropAndResize` Signed-off-by: Yong Tang --- tensorflow/core/kernels/crop_and_resize_op_test.cc | 6 +++++- tensorflow/core/ops/image_ops.cc | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/crop_and_resize_op_test.cc b/tensorflow/core/kernels/crop_and_resize_op_test.cc index 22c659b587..a35e1b0788 100644 --- a/tensorflow/core/kernels/crop_and_resize_op_test.cc +++ b/tensorflow/core/kernels/crop_and_resize_op_test.cc @@ -61,8 +61,12 @@ class CropAndResizeOpTest : public OpsTestBase { REGISTER_TEST(float) REGISTER_TEST(double) -REGISTER_TEST(int8) REGISTER_TEST(uint8) +REGISTER_TEST(uint16) +REGISTER_TEST(int8) +REGISTER_TEST(int16) +REGISTER_TEST(int32) +REGISTER_TEST(int64) #undef REGISTER_TEST diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc index 89c9da81c5..e9bf29d172 100644 --- a/tensorflow/core/ops/image_ops.cc +++ b/tensorflow/core/ops/image_ops.cc @@ -1097,7 +1097,7 @@ REGISTER_OP("CropAndResize") .Input("box_ind: int32") .Input("crop_size: int32") .Output("crops: float") - .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}") + .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}") .Attr("method: {'bilinear'} = 'bilinear'") .Attr("extrapolation_value: float = 0") .SetShapeFn([](InferenceContext* c) { @@ -1204,7 +1204,7 @@ REGISTER_OP("CropAndResizeGradBoxes") .Input("boxes: float") .Input("box_ind: int32") .Output("output: float") - .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}") + .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}") .Attr("method: {'bilinear'} = 'bilinear'") .SetShapeFn([](InferenceContext* c) { c->set_output(0, c->input(2)); -- GitLab From 09ff3f7296a66c39535e097ecb6b82e3fc42ba30 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 12:51:28 -0700 Subject: [PATCH 113/573] Internal change. PiperOrigin-RevId: 172641543 --- tensorflow/python/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e4e284dcdf..cbeb0b46cb 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3685,7 +3685,10 @@ py_test( size = "medium", srcs = ["training/monitored_session_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], + tags = [ + "no_windows", + "notsan", # b/67945581 + ], deps = [ ":array_ops", ":client_testlib", -- GitLab From 38bcb3c02fbc5185d6c1fb7e8327a070284b66e4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 12:53:54 -0700 Subject: [PATCH 114/573] Bug fixes for fold_constants_lib. 1. Tensor names in TF may be in the form of "a:0", "a:1", or "a" as a shorthand notation of "a:0". FoldConstant library always expected the shorthand notation, and did not handle the cases where explicit notation was passed to input or output list. This means that this library could not handle the case when input or output were not the first output of a node. 2. To match the input nodes in the original graph and the added Recv nodes in rewritten graph, FoldConstant library used prefix matching. Unfortunately, this means that when a input name is a prefix of another input name, there is possibility that wrong Recv node gets matched. For example, if input names were "placeholder" and "placeholder_1", then it did not handle the case very well. 3. RemoveUnusedNodes() in FoldConstants lib could remove nodes which output depended on. This happened when an input name points to a node with multiple outputs and not all outputs of that node were included in the input names. 4. ReplaceSendRecvs() in FoldConstants lib assumed that all input nodes are removed during rewriting the graph. This assumption is not necessarily true, and it could add a duplicate node in the graph. PiperOrigin-RevId: 172641947 --- .../graph_transforms/fold_constants_lib.cc | 202 ++++++++---------- .../graph_transforms/fold_constants_test.cc | 85 +++++++- 2 files changed, 175 insertions(+), 112 deletions(-) diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc index 30290c7a16..f2934a79bd 100644 --- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc +++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc @@ -17,12 +17,20 @@ limitations under the License. #include #include +#include +#include +#include +#include +#include +#include #include "tensorflow/core/common_runtime/constant_folding.h" #include "tensorflow/core/common_runtime/shape_refiner.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/graph/subgraph.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/util/command_line_flags.h" @@ -30,33 +38,38 @@ limitations under the License. namespace tensorflow { namespace graph_transforms { +namespace { +using StringPieceSet = std::unordered_set; +template +using StringPieceMap = std::unordered_map; +} // namespace Status ReplaceSendRecvs(const GraphDef& original_graph_def, const GraphDef& rewritten_graph_def, const std::vector& inputs, const std::vector& outputs, GraphDef* output_graph_def) { - std::map original_map; - MapNamesToNodes(original_graph_def, &original_map); - std::map new_node_names; - for (const NodeDef& node : rewritten_graph_def.node()) { - // If the op isn't a Recv, or it was in the original, nothing to do. - if ((node.op() != "_Recv") || (original_map.count(node.name()) == 1)) { - continue; - } - // See if it matches an input from the original. - for (const string& input : inputs) { - // Here we rely on the naming convention for the Recv nodes that - // RewriteGraphForExecution adds in the place of the feed inputs. - string input_prefix = "_recv_" + input + "_"; - if (StringPiece(node.name()).starts_with(input_prefix)) { - // If it does, prepare to rename any inputs that refer to it. - new_node_names[node.name()] = input; - } - } + // recv_node_names serves as a string storage for recv node names. + std::vector recv_node_names(inputs.size()); + StringPieceMap recv_node_map; + StringPieceSet input_nodes; + for (int i = 0; i < inputs.size(); ++i) { + // RewriteGraphForExecution adds a recv node for each input edge. We assume + // here that adding such recv node did not fail. For example, the original + // graph did not already have a node with the name for the new added recv + // node. + TensorId id = ParseTensorName(inputs[i]); + input_nodes.insert(id.first); + string& recv_node_name = recv_node_names[i]; + recv_node_name = strings::StrCat("_recv_", id.first, "_", id.second); + recv_node_map.emplace(recv_node_name, id); + } + + StringPieceMap original_map; + for (const NodeDef& node : original_graph_def.node()) { + original_map.emplace(node.name(), &node); } - std::vector nodes_to_add; for (const NodeDef& node : rewritten_graph_def.node()) { if ((node.op() == "_Send") || (node.op() == "_Recv")) { // If the op is a Send or Recv that wasn't in the original, skip it. @@ -64,55 +77,68 @@ Status ReplaceSendRecvs(const GraphDef& original_graph_def, continue; } } - NodeDef new_node; - new_node = node; - new_node.mutable_input()->Clear(); - for (const string& old_input : node.input()) { - string input_prefix; - string input_node_name; - string input_suffix; - NodeNamePartsFromInput(old_input, &input_prefix, &input_node_name, - &input_suffix); - string new_input; - if (new_node_names.count(input_node_name) > 0) { - new_input = - input_prefix + new_node_names[input_node_name] + input_suffix; - } else { - new_input = old_input; + + NodeDef* new_node = output_graph_def->add_node(); + new_node->MergeFrom(node); + for (int i = 0; i < new_node->input_size(); ++i) { + string& input = *new_node->mutable_input(i); + TensorId id = ParseTensorName(input); + const auto iter = recv_node_map.find(id.first); + if (iter != recv_node_map.end()) { + // The node being substituted is a Recv node, and it has only one + // output. If this input is not a control input, then replace the input + // with the mapped value. Otherwise, replace the node name only. + if (id.second != Graph::kControlSlot) { + CHECK_EQ(id.second, 0); + input = iter->second.ToString(); + } else { + id.first = iter->second.first; + input = id.ToString(); + } } - *(new_node.mutable_input()->Add()) = new_input; } - nodes_to_add.push_back(new_node); - } - for (std::pair entry : new_node_names) { - string removed_node_name = entry.second; - const NodeDef* removed_node = original_map[removed_node_name]; - NodeDef new_node; - new_node = *removed_node; - nodes_to_add.push_back(new_node); + + // RewriteGraphForExecution() did not remove this input node. Remove this + // node name from input_nodes so that a duplicate does not get added to the + // output_graph_def. + auto iter = input_nodes.find(new_node->name()); + if (iter != input_nodes.end()) { + input_nodes.erase(iter); + } } - for (const NodeDef& node : nodes_to_add) { - *output_graph_def->mutable_node()->Add() = node; + // Some input nodes are removed in rewrite_graph_def. Add those nodes to + // output_graph_def. + for (StringPiece name : input_nodes) { + const NodeDef& removed_node = *CHECK_NOTNULL(original_map[name]); + output_graph_def->add_node()->MergeFrom(removed_node); } + return Status::OK(); } Status RemoveUnusedNodes(const GraphDef& input_graph_def, const TransformFuncContext& context, GraphDef* output_graph_def) { - std::map node_map; - MapNamesToNodes(input_graph_def, &node_map); + StringPieceMap node_map; + for (const NodeDef& node : input_graph_def.node()) { + node_map.emplace(node.name(), &node); + } - std::set used_nodes; + std::unordered_set input_names; for (const string& input : context.input_names) { - used_nodes.insert(input); + input_names.insert(ParseTensorName(input)); + } + StringPieceSet used_nodes; + StringPieceSet current_nodes; + for (const string& name : context.output_names) { + TensorId id = ParseTensorName(name); + used_nodes.insert(id.first); + current_nodes.insert(id.first); } - std::vector current_nodes = context.output_names; while (!current_nodes.empty()) { - std::set next_nodes; - for (const string& node_name : current_nodes) { - used_nodes.insert(node_name); + StringPieceSet next_nodes; + for (StringPiece node_name : current_nodes) { if (node_map.count(node_name) == 0) { LOG(ERROR) << "Bad graph structure, no node named '" << node_name << "' found for input lookup"; @@ -120,14 +146,20 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def, node_name, "' found for input lookup"); } const NodeDef& node = *(node_map[node_name]); - for (const string& input_name : node.input()) { - const string& input_node_name = NodeNameFromInput(input_name); - if (used_nodes.count(input_node_name) == 0) { - next_nodes.insert(input_node_name); + for (const string& input : node.input()) { + TensorId id = ParseTensorName(input); + if (input_names.count(id) > 0) { + continue; + } + if (used_nodes.insert(id.first).second) { + next_nodes.insert(id.first); } } } - current_nodes = std::vector(next_nodes.begin(), next_nodes.end()); + current_nodes.swap(next_nodes); + } + for (const TensorId& id : input_names) { + used_nodes.insert(id.first); } FilterGraphDef( input_graph_def, @@ -141,7 +173,7 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def, Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle, shape_inference::InferenceContext* context, PartialTensorShape* shape) { - // The default is already unknown + // The default is already unknown. if (!context->RankKnown(handle)) return Status::OK(); std::vector dims(context->Rank(handle)); @@ -151,47 +183,6 @@ Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle, return PartialTensorShape::MakePartialShape(dims.data(), dims.size(), shape); } -Status ShapeForNode(const TransformFuncContext& context, - const string& node_name, TensorShape* result, - bool* has_shape_specified) { - *has_shape_specified = false; - - // Check to see if we have been given a default for all placeholders. - if (context.params.count("type")) { - if (context.params.at("shape").size() != 1) { - return errors::InvalidArgument( - "You must pass no more than one default 'shape' to " - "fold_constants"); - } - const string& shape_string = context.params.at("shape")[0]; - TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result)); - *has_shape_specified = true; - } - - // See if there's a particular type specified for this placeholder. - if (context.params.count("name") || context.params.count("type_for_name")) { - if (!context.params.count("name") || - !context.params.count("type_for_name") || - (context.params.at("type_for_name").size() != - context.params.at("name").size())) { - return errors::InvalidArgument( - "You must pass a 'shape_for_name' arg for every 'name', e.g. " - "fold_constants(name=foo, shape_for_name=\"2,2,1\", name=bar, " - "shape_for_name=\"1\""); - } - const int name_count = context.params.at("name").size(); - for (int i = 0; i < name_count; ++i) { - if (context.params.at("name")[i] == node_name) { - const string& shape_string = context.params.at("shape_for_name")[i]; - TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result)); - *has_shape_specified = true; - } - } - } - - return Status::OK(); -} - // Converts any sub-graphs that can be resolved into constant expressions into // single Const ops. Status FoldConstants(const GraphDef& input_graph_def, @@ -215,17 +206,6 @@ Status FoldConstants(const GraphDef& input_graph_def, GraphDef cleaned_graph_def; RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def); - // Set specified shapes. - for (NodeDef& node : *cleaned_graph_def.mutable_node()) { - TensorShape shape; - bool has_shape_specified; - TF_RETURN_IF_ERROR( - ShapeForNode(context, node.name(), &shape, &has_shape_specified)); - if (has_shape_specified) { - SetNodeAttr("shape", shape, &node); - } - } - TF_RETURN_IF_ERROR( ImportGraphDef({}, cleaned_graph_def, &input_graph, &shape_refiner)); } else { diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc index fd4188a6a4..41106de008 100644 --- a/tensorflow/tools/graph_transforms/fold_constants_test.cc +++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc @@ -74,6 +74,9 @@ class ConstantFoldingTest : public ::testing::Test { TestConstantFolding(graph_def, {{"placeholder_expect_remains", placeholder_tensor}}, {}, {"output_expect_remains"}, {}); + TestConstantFolding(graph_def, + {{"placeholder_expect_remains:0", placeholder_tensor}}, + {}, {"output_expect_remains:0"}, {}); } void TestOpExclusionAdd() { @@ -256,10 +259,40 @@ class ConstantFoldingTest : public ::testing::Test { EXPECT_EQ(0, node_map.count("new_send")); } + void TestReplaceSendRecvsPrefixNames() { + using namespace ::tensorflow::ops; // NOLINT(build/namespaces) + + auto o_root = tensorflow::Scope::NewRootScope(); + auto a = Placeholder(o_root.WithOpName("placeholder"), DT_FLOAT); + auto b = Placeholder(o_root.WithOpName("placeholder_1"), DT_FLOAT); + auto add_o = Add(o_root.WithOpName("add"), a, b); + GraphDef o_graph_def; + TF_ASSERT_OK(o_root.ToGraphDef(&o_graph_def)); + + auto n_root = tensorflow::Scope::NewRootScope(); + auto c = _Recv(n_root.WithOpName("_recv_placeholder_0"), DT_FLOAT, "", "", + 0, ""); + auto d = _Recv(n_root.WithOpName("_recv_placeholder_1_0"), DT_FLOAT, "", "", + 0, ""); + auto add_n = Add(n_root.WithOpName("add"), c, d); + GraphDef n_graph_def; + TF_ASSERT_OK(n_root.ToGraphDef(&n_graph_def)); + + GraphDef result_graph_def; + TF_ASSERT_OK(graph_transforms::ReplaceSendRecvs( + o_graph_def, n_graph_def, {"placeholder", "placeholder_1"}, {"add"}, + &result_graph_def)); + + std::map node_map; + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(1, node_map.count("placeholder")); + EXPECT_EQ(1, node_map.count("placeholder_1")); + EXPECT_EQ(1, node_map.count("add")); + } + void TestRemoveUnusedNodes() { using namespace ::tensorflow::ops; // NOLINT(build/namespaces) auto root = tensorflow::Scope::NewRootScope(); - using namespace ::tensorflow::ops; // NOLINT(build/namespaces) const int width = 100; @@ -295,6 +328,48 @@ class ConstantFoldingTest : public ::testing::Test { EXPECT_EQ(1, node_map.count("output")); EXPECT_EQ(0, node_map.count("unused")); } + + void TestRemoveUnusedNodesMultipleOutputs() { + using namespace ::tensorflow::ops; // NOLINT(build/namespaces) + auto root = tensorflow::Scope::NewRootScope(); + + // a b + // \ / + // shape_n + // \ / + // c + auto a = Placeholder(root.WithOpName("a"), DT_FLOAT); + auto b = Placeholder(root.WithOpName("b"), DT_FLOAT); + auto shape_n = ShapeN(root.WithOpName("shape_n"), {Output(a), Output(b)}); + auto c = Add(root.WithOpName("c"), shape_n[0], shape_n[1]); + + GraphDef graph_def; + TF_ASSERT_OK(root.ToGraphDef(&graph_def)); + GraphDef result_graph_def; + TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes( + graph_def, {{shape_n[0].name()}, {"c"}}, &result_graph_def)); + + // Only one output of shape_n node is fed input. Hence the graph search + // should propagate to inputs of shape_n. Nothing to remove here. + std::map node_map; + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(1, node_map.count("a")); + EXPECT_EQ(1, node_map.count("b")); + EXPECT_EQ(1, node_map.count("c")); + + result_graph_def.Clear(); + TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes( + graph_def, {{shape_n[0].name(), shape_n[1].name()}, {"c"}}, + &result_graph_def)); + + // Both outputs of shape_n node are fed inputs. shape_n does not function + // and inputs to shape_n should be removed. + node_map.clear(); + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(0, node_map.count("a")); + EXPECT_EQ(0, node_map.count("b")); + EXPECT_EQ(1, node_map.count("c")); + } }; TEST_F(ConstantFoldingTest, TestSimpleAdd) { TestSimpleAdd(); } @@ -309,7 +384,15 @@ TEST_F(ConstantFoldingTest, TestPreserveOutputShapes) { TEST_F(ConstantFoldingTest, TestReplaceSendRecvs) { TestReplaceSendRecvs(); } +TEST_F(ConstantFoldingTest, TestReplaceSendRecvsPrefixNames) { + TestReplaceSendRecvsPrefixNames(); +} + TEST_F(ConstantFoldingTest, TestRemoveUnusedNodes) { TestRemoveUnusedNodes(); } +TEST_F(ConstantFoldingTest, TestRemoveUnusedNodesMultipleOutputs) { + TestRemoveUnusedNodesMultipleOutputs(); +} + } // namespace graph_transforms } // namespace tensorflow -- GitLab From b7e85339b286f34f215cca3dcb700dbd8f276de3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 13:02:53 -0700 Subject: [PATCH 115/573] Adds visibility to sgdr_learning_rate_decay. Currently SGD with warm restarts is siloed in tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py, since it is not listed in the 'training_py' build filegroup. This change simply adds sgdr_learning_rate_decay to this filegroup so that other projects can use warm restarts during optimization. PiperOrigin-RevId: 172643218 --- tensorflow/contrib/training/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD index 80a5debe99..0df5ff50c0 100644 --- a/tensorflow/contrib/training/BUILD +++ b/tensorflow/contrib/training/BUILD @@ -26,6 +26,7 @@ py_library( "python/training/resample.py", "python/training/sampling_ops.py", "python/training/sequence_queueing_state_saver.py", + "python/training/sgdr_learning_rate_decay.py", "python/training/training.py", "python/training/tuner.py", ], -- GitLab From f6968a25c9dfc962851806d094dc98f5b502a4f9 Mon Sep 17 00:00:00 2001 From: Jianwei Xie Date: Wed, 18 Oct 2017 13:07:01 -0700 Subject: [PATCH 116/573] Add logging verbosity to mnist.py PiperOrigin-RevId: 172643922 --- tensorflow/examples/learn/mnist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py index 5344526b52..88425ea0d0 100644 --- a/tensorflow/examples/learn/mnist.py +++ b/tensorflow/examples/learn/mnist.py @@ -97,6 +97,8 @@ def conv_model(features, labels, mode): def main(unused_args): + tf.logging.set_verbosity(tf.logging.INFO) + ### Download and load MNIST dataset. mnist = tf.contrib.learn.datasets.DATASETS['mnist']('/tmp/mnist') train_input_fn = tf.estimator.inputs.numpy_input_fn( @@ -115,6 +117,7 @@ def main(unused_args): feature_columns = [ tf.feature_column.numeric_column( X_FEATURE, shape=mnist.train.images.shape[1:])] + classifier = tf.estimator.LinearClassifier( feature_columns=feature_columns, n_classes=N_DIGITS) classifier.train(input_fn=train_input_fn, steps=200) -- GitLab From 08aeb0f960329efa7f477fd184d2e676a96da415 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Wed, 18 Oct 2017 13:22:01 -0700 Subject: [PATCH 117/573] Automated g4 rollback of changelist 172336111 PiperOrigin-RevId: 172645893 --- tensorflow/python/framework/ops.py | 11 ++- .../resource_variable_ops_test.py | 10 +++ .../python/ops/resource_variable_ops.py | 33 ++++++++ tensorflow/python/training/adam_test.py | 81 ++++++++++--------- tensorflow/python/training/saver_test.py | 47 +++++------ 5 files changed, 117 insertions(+), 65 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 75750ecd5a..85b875aa3a 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -2517,7 +2517,14 @@ class Graph(object): # A map from tensor handle to its delete op. self._handle_deleters = {} # Resource container. - self._container = "" + if context.in_graph_mode(): + self._container_prefix = "" + else: + # In Eager mode, isolate resources (particularly ResourceVariables) in + # Graphs by default. This prevents unintended variable sharing. Graph mode + # gets this kind of isolation from Sessions. + self._container_prefix = "eager-execution-%d/" % (uid(),) + self._container = self._container_prefix self._registered_ops = op_def_registry.get_registered_ops() # TODO(skyewm): fold as much of the above as possible into the C @@ -3829,7 +3836,7 @@ class Graph(object): """ original_container = self._container try: - self._container = container_name + self._container = self._container_prefix + container_name yield self._container finally: self._container = original_container diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index ec9192b1a0..23676223dc 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -428,6 +428,16 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(1, v1.read_value().numpy()) self.assertEqual(2, v2.read_value().numpy()) + def testDestruction(self): + with context.eager_mode(): + var = resource_variable_ops.ResourceVariable(initial_value=1.0, + name="var8") + var.__del__() + with self.assertRaisesRegexp(errors.NotFoundError, + r"Resource .*\/var8\/.* does not exist."): + resource_variable_ops.destroy_resource_op(var._handle, + ignore_lookup_error=False) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 2c9a3ff19a..dd3f167145 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -427,6 +427,39 @@ class ResourceVariable(variables.Variable): self._constraint = None # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py) + def __del__(self): + if not self._in_graph_mode: + # There is only one ResourceVariable object for each underlying resource + # (cached in the Graph's VariableStore when created with get_variable), so + # it is safe to delete the resource we have a handle to. Each Graph has a + # unique container name in Eager, which prevents resource sharing. + # + # The Graph's VariableStore contains strong references to ResourceVariable + # objects created with get_variable, so this destructor will only be + # callled once the Graph is garbage collected for those objects. However, + # explicitly created ResourceVariables (e.g. through tfe.Variable) may be + # collected earlier. + try: + # We have checked that this ResourceVariable was created in Eager + # mode. However, this destructor may be running in graph mode + # (especially during unit tests). To clean up successfully, we switch + # back into Eager temporarily. + with context.eager_mode(): + with ops.device(self._handle_device): + gen_resource_variable_ops.destroy_resource_op( + self._handle, ignore_lookup_error=True) + except TypeError: + # Suppress some exceptions, mainly for the case when we're running on + # module deletion. Things that can go wrong include the context module + # already being unloaded, self._handle._handle_data no longer being + # valid, and so on. Printing warnings in these cases is silly + # (exceptions raised from __del__ are printed as warnings to stderr). + pass # 'NoneType' object is not callable when the handle has been + # partially unloaded. + except AttributeError: + pass # 'NoneType' object has no attribute 'eager_mode' when context has + # been unloaded. Will catch other module unloads as well. + @property def dtype(self): """The dtype of this variable.""" diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py index defcf33714..176d20bd60 100644 --- a/tensorflow/python/training/adam_test.py +++ b/tensorflow/python/training/adam_test.py @@ -152,53 +152,54 @@ class AdamOptimizerTest(test.TestCase): def doTestBasic(self, use_resource=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = adam.AdamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + with self.test_session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - if context.in_graph_mode(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + if use_resource: + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) - beta1_power, beta2_power = opt._get_beta_accumulators() + opt = adam.AdamOptimizer() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - # Run 3 steps of Adam - for t in range(1, 4): if context.in_graph_mode(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**(t + 1), - self.evaluate(beta2_power)) + beta1_power, beta2_power = opt._get_beta_accumulators() - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + # Run 3 steps of Adam + for t in range(1, 4): + if context.in_graph_mode(): + self.evaluate(update) + elif t > 1: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta2_power)) + + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testBasic(self): with self.test_session(): diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index 07cd67a4b9..aeb8eaffe8 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -479,14 +479,14 @@ class SaverTest(test.TestCase): self.assertEqual(30.0, v2_2.values().eval()) def _SaveAndLoad(self, var_name, var_value, other_value, save_path): - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: var = resource_variable_ops.ResourceVariable(var_value, name=var_name) save = saver_module.Saver({var_name: var}) if context.in_graph_mode(): self.evaluate(var.initializer) val = save.save(sess, save_path) self.assertEqual(save_path, val) - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: var = resource_variable_ops.ResourceVariable(other_value, name=var_name) save = saver_module.Saver({var_name: var}) save.restore(sess, save_path) @@ -619,27 +619,28 @@ class SaverTest(test.TestCase): # Save and reload one Variable named "var0". self._SaveAndLoad("var0", 0.0, 1.0, save_path) for use_tensor in [True, False]: - var = resource_variable_ops.ResourceVariable(1.0, name="var0") - save = saver_module.Saver( - { - var._shared_name: var - }, pad_step_number=pad_step_number) - if context.in_graph_mode(): - self.evaluate(var.initializer) - sess = ops_lib.get_default_session() - else: - sess = None - if use_tensor: - global_step = constant_op.constant(global_step_int) - val = save.save(sess, save_path, global_step=global_step) - else: - val = save.save(sess, save_path, global_step=global_step_int) - if pad_step_number: - expected_save_path = "%s-%s" % (save_path, - "{:08d}".format(global_step_int)) - else: - expected_save_path = "%s-%d" % (save_path, global_step_int) - self.assertEqual(expected_save_path, val) + with self.test_session(graph=ops_lib.Graph()): + var = resource_variable_ops.ResourceVariable(1.0, name="var0") + save = saver_module.Saver( + { + var._shared_name: var + }, pad_step_number=pad_step_number) + if context.in_graph_mode(): + self.evaluate(var.initializer) + sess = ops_lib.get_default_session() + else: + sess = None + if use_tensor: + global_step = constant_op.constant(global_step_int) + val = save.save(sess, save_path, global_step=global_step) + else: + val = save.save(sess, save_path, global_step=global_step_int) + if pad_step_number: + expected_save_path = "%s-%s" % (save_path, + "{:08d}".format(global_step_int)) + else: + expected_save_path = "%s-%d" % (save_path, global_step_int) + self.assertEqual(expected_save_path, val) def testSaveWithGlobalStepWithPadding(self): self.testSaveWithGlobalStep(pad_step_number=True) -- GitLab From d65f7b9077b7191d3aa3f1183b6b119c480faa05 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 13:26:24 -0700 Subject: [PATCH 118/573] Correct the docstring to reflect that the values of cols_to_vars are always lists of Variable's (never single Variable's or PartitionedVariables), and make this true for bias. PiperOrigin-RevId: 172646456 --- tensorflow/python/feature_column/BUILD | 1 + .../python/feature_column/feature_column.py | 33 +++++----- .../feature_column/feature_column_test.py | 61 +++++++++++++++++-- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD index 27062adb61..b1c81dd58c 100644 --- a/tensorflow/python/feature_column/BUILD +++ b/tensorflow/python/feature_column/BUILD @@ -86,6 +86,7 @@ py_test( "//tensorflow/python:framework_ops", "//tensorflow/python:lookup_ops", "//tensorflow/python:parsing_ops", + "//tensorflow/python:partitioned_variables", "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 81f4f45fcb..190a25d4d7 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -197,12 +197,13 @@ def input_layer(features, trainable: If `True` also add the variable to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). cols_to_vars: If not `None`, must be a dictionary that will be filled with a - mapping from `_FeatureColumn` to associated `Variable` (or list of - `Variable`, or `PartitionedVariable`. For example, after the call, we - might have cols_to_vars = {_EmbeddingColumn( + mapping from `_FeatureColumn` to list of `Variable`s. For example, after + the call, we might have cols_to_vars = + {_EmbeddingColumn( categorical_column=_HashedCategoricalColumn( key='sparse_feature', hash_bucket_size=5, dtype=tf.string), - dimension=10): [, - 'bias': , + [], + 'bias': [], _NumericColumn( key='numeric_feature2', shape=(2,)): - } - Note that it will also contain a string key 'bias'. If a column creates - no variables, its value will be an empty list. + []} + If a column creates no variables, its value will be an empty list. Note + that cols_to_vars will also contain a string key 'bias' that maps to a + list of Variables. Returns: A `Tensor` which represents predictions/logits of a linear model. Its shape @@ -366,8 +367,12 @@ def linear_model(features, predictions = nn_ops.bias_add( predictions_no_bias, bias, name='weighted_sum') if cols_to_vars is not None: - # Add the bias to cols_to_vars as well. - cols_to_vars['bias'] = bias + # Add the bias to cols_to_vars as well, converting the Variable or + # PartitionedVariable to a list of Variable's. + if isinstance(bias, variables.Variable): + cols_to_vars['bias'] = [bias] + else: # Must be a PartitionedVariable. + cols_to_vars['bias'] = list(bias) return predictions diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 112600439b..e57e9a9836 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -41,6 +41,7 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import parsing_ops +from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test @@ -1354,10 +1355,33 @@ class LinearModelTest(test.TestCase): bias = get_linear_model_bias() price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) - self.assertEqual(cols_to_vars['bias'], bias) + self.assertAllEqual(cols_to_vars['bias'], [bias]) self.assertAllEqual(cols_to_vars[price1], [price1_var]) self.assertAllEqual(cols_to_vars[price2], [price2_var]) + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1', shape=2) + price2 = fc.numeric_column('price2', shape=3) + with ops.Graph().as_default(): + features = { + 'price1': [[1., 2.], [6., 7.]], + 'price2': [[3., 4., 5.], [8., 9., 10.]] + } + cols_to_vars = {} + with variable_scope.variable_scope( + 'linear', + partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)): + fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars) + with _initialized_session(): + self.assertEqual([0.], cols_to_vars['bias'][0].eval()) + # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables. + self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval()) + # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and + # a [1, 1] Variable. + self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval()) + def test_dense_collection(self): price = fc.numeric_column('price') with ops.Graph().as_default() as g: @@ -1761,9 +1785,38 @@ class InputLayerTest(test.TestCase): self.assertEqual(0, len(cols_to_vars[price1])) self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized])) self.assertEqual(1, len(cols_to_vars[some_embedding_column])) - for var in cols_to_vars[some_embedding_column]: - self.assertIsInstance(var, variables_lib.Variable) - self.assertAllEqual(var.shape, [5, 10]) + self.assertIsInstance(cols_to_vars[some_embedding_column][0], + variables_lib.Variable) + self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10]) + + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1') + dense_feature = fc.numeric_column('dense_feature') + dense_feature_bucketized = fc.bucketized_column( + dense_feature, boundaries=[0.]) + some_sparse_column = fc.categorical_column_with_hash_bucket( + 'sparse_feature', hash_bucket_size=5) + some_embedding_column = fc.embedding_column( + some_sparse_column, dimension=10) + with ops.Graph().as_default(): + features = { + 'price1': [[3.], [4.]], + 'dense_feature': [[-1.], [4.]], + 'sparse_feature': [['a'], ['x']], + } + cols_to_vars = {} + all_cols = [price1, dense_feature_bucketized, some_embedding_column] + with variable_scope.variable_scope( + 'input_from_feature_columns', + partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0)): + fc.input_layer(features, all_cols, cols_to_vars=cols_to_vars) + self.assertItemsEqual(list(cols_to_vars.keys()), all_cols) + self.assertEqual(0, len(cols_to_vars[price1])) + self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized])) + self.assertEqual(3, len(cols_to_vars[some_embedding_column])) + self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [2, 10]) + self.assertAllEqual(cols_to_vars[some_embedding_column][1].shape, [2, 10]) + self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10]) def test_column_order(self): price_a = fc.numeric_column('price_a') -- GitLab From 5565aac7876a1eacdae29ac24d95c0e94c9062c6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 13:33:29 -0700 Subject: [PATCH 119/573] Changes MultiLabelHead.create_loss to return a Tensor of size [batch_size, 1], to be consistent with other heads. PiperOrigin-RevId: 172647355 --- .../estimator/python/estimator/head.py | 16 +++++----- .../estimator/python/estimator/head_test.py | 32 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index f8648fe5bf..ebf91e8bb4 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -265,6 +265,9 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access unweighted_loss = losses.sigmoid_cross_entropy( multi_class_labels=processed_labels, logits=logits, reduction=losses.Reduction.NONE) + # Averages loss over classes. + unweighted_loss = math_ops.reduce_mean( + unweighted_loss, axis=-1, keep_dims=True) return head_lib.LossAndLabels( unweighted_loss=unweighted_loss, processed_labels=processed_labels) @@ -294,12 +297,9 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) - # Averages loss over classes. - per_example_loss = math_ops.reduce_mean( - unweighted_loss, axis=-1, keep_dims=True) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( - per_example_loss, weights=weights, reduction=losses.Reduction.SUM) + unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, @@ -309,7 +309,7 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access labels=processed_labels, probabilities=probabilities, weights=weights, - per_example_loss=per_example_loss)) + unweighted_loss=unweighted_loss)) # Train. if train_op_fn is None: @@ -330,16 +330,16 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access loss=training_loss, train_op=train_op_fn(training_loss)) - def _eval_metric_ops(self, labels, probabilities, weights, per_example_loss): + def _eval_metric_ops(self, labels, probabilities, weights, unweighted_loss): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope( - None, 'metrics', [labels, probabilities, weights, per_example_loss]): + None, 'metrics', [labels, probabilities, weights, unweighted_loss]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( - per_example_loss, weights=weights, name=keys.LOSS_MEAN), + unweighted_loss, weights=weights, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc( labels=labels, predictions=probabilities, weights=weights, diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py index dcbe62b497..ec1386af34 100644 --- a/tensorflow/contrib/estimator/python/estimator/head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/head_test.py @@ -80,9 +80,13 @@ def _sigmoid(logits): def _sigmoid_cross_entropy(labels, logits): + """Returns sigmoid cross entropy averaged over classes.""" sigmoid_logits = _sigmoid(logits) - return (-labels * np.log(sigmoid_logits) - -(1 - labels) * np.log(1 - sigmoid_logits)) + unreduced_result = ( + -labels * np.log(sigmoid_logits) + -(1 - labels) * np.log(1 - sigmoid_logits)) + # Mean over classes + return np.mean(unreduced_result, axis=-1, keepdims=True) class MultiLabelHead(test.TestCase): @@ -226,7 +230,7 @@ class MultiLabelHead(test.TestCase): # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits expected_unweighted_loss = np.array( - [[10., 10.], [15., 0.]], dtype=np.float32) + [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32) actual_unweighted_loss, _ = head.create_loss( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.EVAL, @@ -311,10 +315,8 @@ class MultiLabelHead(test.TestCase): labels = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. - expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes - ) + # Sum over examples. + expected_loss = np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) keys = metric_keys.MetricKeys expected_metrics = { # Average loss over examples. @@ -343,10 +345,9 @@ class MultiLabelHead(test.TestCase): labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) / - n_classes + np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) ) keys = metric_keys.MetricKeys expected_metrics = { @@ -377,10 +378,9 @@ class MultiLabelHead(test.TestCase): labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) / - n_classes + np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) ) keys = metric_keys.MetricKeys expected_metrics = { @@ -407,9 +407,9 @@ class MultiLabelHead(test.TestCase): labels = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes + np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) ) keys = metric_keys.MetricKeys @@ -506,7 +506,7 @@ class MultiLabelHead(test.TestCase): # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits expected_unweighted_loss = np.array( - [[10., 10.], [15., 0.]], dtype=np.float32) + [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32) actual_unweighted_loss, _ = head.create_loss( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.TRAIN, -- GitLab From cadcda216ec7d6f5f3e36dfc7863634f4f03f71f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 14:03:07 -0700 Subject: [PATCH 120/573] Implementation of the Swish activation function. PiperOrigin-RevId: 172651500 --- tensorflow/python/ops/nn.py | 1 + tensorflow/python/ops/nn_impl.py | 42 +++++++++++++++++++ tensorflow/python/ops/nn_test.py | 27 ++++++++++++ .../tools/api/golden/tensorflow.nn.pbtxt | 4 ++ 4 files changed, 74 insertions(+) diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index a80662c8b5..79af3ac117 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -21,6 +21,7 @@ See the @{$python/nn} guide. @@relu @@relu6 @@crelu +@@swish @@elu @@leaky_relu @@selu diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index db8e92831e..2c83e4e29f 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -22,6 +22,7 @@ import math from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import candidate_sampling_ops @@ -269,6 +270,47 @@ def relu_layer(x, weights, biases, name=None): return nn_ops.relu(xw_plus_b, name=name) +def _swish_shape(op): + """Shape helper function for swish and _swish_grad function below.""" + return [op.inputs[0].shape] + + +# Set noinline=True so that sigmoid(features) is re-computed during +# backprop, and we can free the sigmoid(features) expression immediately +# after use during the forward pass. +@function.Defun(shape_func=_swish_shape, func_name="swish_grad", noinline=True) +def _swish_grad(features, grad): + """Gradient of Swish function defined below.""" + sigmoid_features = math_ops.sigmoid(features) + activation_grad = ( + sigmoid_features * (1.0 + features * (1.0 - sigmoid_features))) + return grad * activation_grad + + +@function.Defun( + grad_func=_swish_grad, + shape_func=_swish_shape, + func_name="swish", + noinline=True) +def swish(features): + # pylint: disable=g-doc-args + """Computes the Swish activation function: `x * sigmoid(x)`. + + Source: "Swish: a Self-Gated Activation Function" (Ramachandran et al. 2017) + https://arxiv.org/abs/1710.05941 + + Args: + features: A `Tensor` representing preactivation values. + name: A name for the operation (optional). + + Returns: + The activation value. + """ + # pylint: enable=g-doc-args + features = ops.convert_to_tensor(features, name="features") + return features * math_ops.sigmoid(features) + + def l2_normalize(x, dim, epsilon=1e-12, name=None): """Normalizes along dimension `dim` using an L2 norm. diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index 3528b60ca7..3b918e4f74 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_impl from tensorflow.python.ops import nn_ops from tensorflow.python.ops import partitioned_variables @@ -860,6 +861,32 @@ class LeakyReluTest(test_lib.TestCase): self.assertAllClose(outputs, [-0.2, 0.0, 0.5, 1.0, 2.0]) +class SwishTest(test_lib.TestCase): + + def testValues(self): + np_values = np.array( + [np.linspace(-10.0, 0.0, 100), + np.linspace(0.0, 10.0, 100)], + dtype=np.float32) + tf_values = constant_op.constant(np_values) + actual_tf_outputs = nn_impl.swish(tf_values) + expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values) + with self.test_session() as sess: + actual_outputs, expected_outputs = sess.run( + [actual_tf_outputs, expected_tf_outputs]) + self.assertAllClose(actual_outputs, expected_outputs) + + def testGradients(self): + shape = [5, 3, 4] + sigma = 5 + input_values = np.random.randn(*shape) * sigma + x_tf = constant_op.constant(input_values) + y_tf = nn_impl.swish(x_tf) + with self.test_session(): + err = gradient_checker.compute_gradient_error(x_tf, shape, y_tf, shape) + self.assertLess(err, 1e-4) + + class MomentsTest(test_lib.TestCase): def doOutputTest(self, input_shape, moments_axes, tol=1e-4, diff --git a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt index f10299377b..11637814a6 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt @@ -4,6 +4,10 @@ tf_module { name: "rnn_cell" mtype: "" } + member { + name: "swish" + mtype: "" + } member_method { name: "all_candidate_sampler" argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " -- GitLab From 71cea5ba4eafabb4a5515025bd1b6106faa0c958 Mon Sep 17 00:00:00 2001 From: Jianwei Xie Date: Wed, 18 Oct 2017 14:06:53 -0700 Subject: [PATCH 121/573] Modify the learn examples wide_and_deep to use tf.estimator.train_and_evaluate. PiperOrigin-RevId: 172652065 --- .../examples/learn/wide_n_deep_tutorial.py | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/tensorflow/examples/learn/wide_n_deep_tutorial.py b/tensorflow/examples/learn/wide_n_deep_tutorial.py index 7b9381311c..e447b3e24e 100644 --- a/tensorflow/examples/learn/wide_n_deep_tutorial.py +++ b/tensorflow/examples/learn/wide_n_deep_tutorial.py @@ -107,6 +107,9 @@ deep_columns = [ ] +FLAGS = None + + def maybe_download(train_data, test_data): """Maybe downloads training data and returns train and test file names.""" if train_data: @@ -154,7 +157,14 @@ def build_estimator(model_dir, model_type): def input_fn(data_file, num_epochs, shuffle): - """Input builder function.""" + """Returns an `input_fn` required by Estimator train/evaluate. + + Args: + data_file: The file path to the dataset. + num_epochs: Number of epochs to iterate over data. If `None`, `input_fn` + will generate infinite stream of data. + shuffle: bool, whether to read the data in random order. + """ df_data = pd.read_csv( tf.gfile.Open(data_file), names=CSV_COLUMNS, @@ -164,43 +174,42 @@ def input_fn(data_file, num_epochs, shuffle): # remove NaN elements df_data = df_data.dropna(how="any", axis=0) labels = df_data["income_bracket"].apply(lambda x: ">50K" in x).astype(int) + return tf.estimator.inputs.pandas_input_fn( x=df_data, y=labels, batch_size=100, num_epochs=num_epochs, shuffle=shuffle, - num_threads=5) + num_threads=1) -def train_and_eval(model_dir, model_type, train_steps, train_data, test_data): - """Train and evaluate the model.""" - train_file_name, test_file_name = maybe_download(train_data, test_data) +def main(_): + tf.logging.set_verbosity(tf.logging.INFO) + + train_file_name, test_file_name = maybe_download(FLAGS.train_data, + FLAGS.test_data) + # Specify file path below if want to find the output easily - model_dir = tempfile.mkdtemp() if not model_dir else model_dir + model_dir = FLAGS.model_dir if FLAGS.model_dir else tempfile.mkdtemp() - m = build_estimator(model_dir, model_type) - # set num_epochs to None to get infinite stream of data. - m.train( + estimator = build_estimator(model_dir, FLAGS.model_type) + + # `tf.estimator.TrainSpec`, `tf.estimator.EvalSpec`, and + # `tf.estimator.train_and_evaluate` API are available in TF 1.4. + train_spec = tf.estimator.TrainSpec( input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True), - steps=train_steps) - # set steps to None to run evaluation until all data consumed. - results = m.evaluate( + max_steps=FLAGS.train_steps) + + eval_spec = tf.estimator.EvalSpec( input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False), + # set steps to None to run evaluation until all data consumed. steps=None) - print("model directory = %s" % model_dir) - for key in sorted(results): - print("%s: %s" % (key, results[key])) - # Manual cleanup - shutil.rmtree(model_dir) - - -FLAGS = None + tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) -def main(_): - train_and_eval(FLAGS.model_dir, FLAGS.model_type, FLAGS.train_steps, - FLAGS.train_data, FLAGS.test_data) + # Manual cleanup + shutil.rmtree(model_dir) if __name__ == "__main__": -- GitLab From 8548e18647d4e574dbf697a81f844b2f0d89bacb Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Wed, 18 Oct 2017 14:15:45 -0700 Subject: [PATCH 122/573] Fix typo in error message for set_caching_device PiperOrigin-RevId: 172653499 --- tensorflow/python/ops/variable_scope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 4614110ba6..22048a0cef 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -999,7 +999,7 @@ class VariableScope(object): def set_caching_device(self, caching_device): """Set caching_device for this scope.""" if context.in_eager_mode(): - raise NotImplementedError("Partitioned variables are not yet supported " + raise NotImplementedError("Caching devices are not yet supported " "in Eager mode.") self._caching_device = caching_device -- GitLab From fae8ee3ae5f758e3f6eec33ec01b933084c5d080 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Wed, 18 Oct 2017 14:47:59 -0700 Subject: [PATCH 123/573] Modifies unsupported properties of EagerTensor to raise AttributeErrors instead of NotImplementedErrors. PiperOrigin-RevId: 172658409 --- tensorflow/python/framework/ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 85b875aa3a..ef0ed8fc53 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -741,19 +741,19 @@ class _EagerTensorBase(Tensor): # Methods not supported / implemented for Eager Tensors. @property def op(self): - raise NotImplementedError("op not supported for Eager Tensors.") + raise AttributeError("op not supported for Eager Tensors.") @property def graph(self): - raise NotImplementedError("graph not supported for Eager Tensors.") + raise AttributeError("graph not supported for Eager Tensors.") @property def name(self): - raise NotImplementedError("name not supported for Eager Tensors.") + raise AttributeError("name not supported for Eager Tensors.") @property def value_index(self): - raise NotImplementedError("value_index not supported for Eager Tensors.") + raise AttributeError("value_index not supported for Eager Tensors.") def consumers(self): raise NotImplementedError("consumers not supported for Eager Tensors.") -- GitLab From b2f5acd2c3fbcccb580d6393c0ce77a32ad01279 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Wed, 18 Oct 2017 15:06:19 -0700 Subject: [PATCH 124/573] Fix a memory leak in graph compiler. - Fix a memory leak in graph compiler by using better memory management. - Simplify the code as I know understand more assumptions of this part of the stack. PiperOrigin-RevId: 172661754 --- tensorflow/compiler/tf2xla/graph_compiler.cc | 46 +++++++------------ tensorflow/compiler/tf2xla/graph_compiler.h | 25 +++++----- .../xla_jit_compiled_cpu_function_test.cc | 14 ++++++ 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc index 6f2f59d98f..9893afa7a0 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.cc +++ b/tensorflow/compiler/tf2xla/graph_compiler.cc @@ -84,7 +84,7 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph, } } // namespace Status GraphCompiler::Compile() { - std::vector bindings(graph_->num_node_ids()); + OutputRegistry output_registry(graph_->num_node_ids()); std::vector topo_sorted_nodes; // XLA requires determinism, generate a stable ordering from DFS. GetReversePostOrder(*graph_, &topo_sorted_nodes, @@ -94,30 +94,23 @@ Status GraphCompiler::Compile() { PartiallySetupParams(¶ms); for (Node* n : topo_sorted_nodes) { - // Set up bindings. - NodeBinding& binding = bindings[n->id()]; - binding.node = n; - Status s = flib_->CreateKernel(n->def(), &binding.op_kernel); - binding.output_attrs.resize(n->num_outputs()); + NodeOutputs node_outputs; + OpKernel* op_kernel_raw = nullptr; + Status s = flib_->CreateKernel(n->def(), &op_kernel_raw); + // Transfer ownership of the kernel to a local smart pointer. + std::unique_ptr op_kernel(op_kernel_raw); + if (!s.ok()) { - binding.op_kernel = nullptr; s = AttachDef(s, *n); LOG(ERROR) << "Executor failed to create kernel. " << s; return s; } - } - - // Bindings are initialized by the size of graph_->num_node_ids. However, the - // graph may contain dead nodes that still hold a valid node id. Thus - // graph_->num_node_ids could be larger than number of topo sorted nodes. - TF_RET_CHECK(bindings.size() >= topo_sorted_nodes.size()); - for (Node* n : topo_sorted_nodes) { TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch()) << "Not supported node: " << n->DebugString(); - NodeBinding& binding = bindings[n->id()]; - params.op_kernel = binding.op_kernel; - params.output_attr_array = binding.output_attrs.data(); + params.op_kernel = op_kernel.get(); + gtl::InlinedVector output_attr(n->num_outputs()); + params.output_attr_array = output_attr.data(); // tensor_inputs_ is a buffer reused across graph traversal. We clean up and // reinitialize the buffer before we visit a new node. @@ -128,8 +121,10 @@ Status GraphCompiler::Compile() { for (auto* e : n->in_edges()) { if (e->IsControlEdge()) continue; Node* src = e->src(); - tensor_inputs_[e->dst_input()] = - bindings[src->id()].tensor_values[e->src_output()]; + TF_RET_CHECK(src->id() < output_registry.size()); + const NodeOutputs& outputs = output_registry[src->id()]; + + tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()]; } OpKernelContext op_context(¶ms, n->num_outputs()); @@ -150,17 +145,8 @@ Status GraphCompiler::Compile() { (*op_context.is_output_dead() ? "(dead)" : ""), SummarizeNode(*n)); } - binding.tensor_values.push_back(tensor_val); - } - } - - // Clean up tensor data and op kernels. - for (NodeBinding& binding : bindings) { - delete binding.op_kernel; - for (auto& t : binding.tensor_values) { - if (!t.is_ref()) { - delete t.tensor; - } + // Set up outputs + output_registry[n->id()].values.push_back(tensor_val); } } return Status::OK(); diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h index ccf9351642..33781d2c21 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.h +++ b/tensorflow/compiler/tf2xla/graph_compiler.h @@ -69,20 +69,23 @@ class GraphCompiler { Status Compile(); private: - // NodeBinding is a wrapper on a `Node` that also contains computed - // TensorValue. - struct NodeBinding { - const Node* node; - // Kernel for this node, to be filled by CreateKernel. - // TODO(yunxing): Switching this to unique_ptr and understand why it crashes - // on GPU devices. - OpKernel* op_kernel; + // NodeOutputs is a wrapper over TensorValues that represents outputs of a + // node. + struct NodeOutputs { + ~NodeOutputs() { + for (auto& v : values) { + CHECK(!v.is_ref()); + delete v.tensor; + } + } + // Output values of this node. - std::vector tensor_values; - // Attributes of the outputs. - gtl::InlinedVector output_attrs; + std::vector values; }; + // A mapping from node id to node output. + using OutputRegistry = std::vector; + // Partially sets params. This partially set params can be reused // across multple nodes visit. void PartiallySetupParams(OpKernelContext::Params* params); diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc index 5bee68eefc..6d49298a6f 100644 --- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc +++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc @@ -129,5 +129,19 @@ TEST(XlaJitCompiledCpuFunction, Sum) { EXPECT_TRUE(ShapeUtil::Compatible(result0, s32)); } +// Test when a graph compilation terminates early, resources are properly +// reclaimed. +TEST(XlaJitCompiledCpuFunction, SumWithJunkAttr) { + GraphDef graph_def = SumGraph(); + + (*graph_def.mutable_node(2)->mutable_attr())["junk"] = + TypeAttrValue(DT_INT32); + + tf2xla::Config config = SumConfig(); + EXPECT_FALSE(XlaJitCompiledCpuFunction::Compile(graph_def, config, + xla::ExecutableBuildOptions()) + .ok()); +} + } // namespace } // namespace tensorflow -- GitLab From 818644d0937d1e4b097b15d8c823835baba9fbc7 Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Wed, 18 Oct 2017 15:07:43 -0700 Subject: [PATCH 125/573] Add tf.contrib.distributions.bijectors.MaskedAutoregressiveFlow. PiperOrigin-RevId: 172662078 --- tensorflow/contrib/distributions/BUILD | 16 + .../bijectors/masked_autoregressive_test.py | 153 ++++++ .../python/ops/bijectors/__init__.py | 5 + .../ops/bijectors/masked_autoregressive.py | 33 ++ .../bijectors/masked_autoregressive_impl.py | 473 ++++++++++++++++++ .../distributions/python/ops/test_util.py | 39 +- 6 files changed, 703 insertions(+), 16 deletions(-) create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index 825ec652d0..1305c28012 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -854,6 +854,22 @@ cuda_py_test( ], ) +cuda_py_test( + name = "masked_autoregressive_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/masked_autoregressive_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "permute_test", size = "small", diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py new file mode 100644 index 0000000000..98c09545ac --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py @@ -0,0 +1,153 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for MaskedAutoregressiveFlow.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.distributions.python.ops import test_util +from tensorflow.contrib.distributions.python.ops.bijectors.invert import Invert +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import masked_autoregressive_default_template +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import MaskedAutoregressiveFlow +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive_impl import _gen_mask +from tensorflow.python.framework import constant_op +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variables +from tensorflow.python.ops.distributions import normal as normal_lib +from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib +from tensorflow.python.platform import test + + +class GenMaskTest(test.TestCase): + + def test346Exclusive(self): + expected_mask = np.array( + [[0, 0, 0, 0], + [0, 0, 0, 0], + [1, 0, 0, 0], + [1, 0, 0, 0], + [1, 1, 0, 0], + [1, 1, 0, 0]]) + mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="exclusive") + self.assertAllEqual(expected_mask, mask) + + def test346Inclusive(self): + expected_mask = np.array( + [[1, 0, 0, 0], + [1, 0, 0, 0], + [1, 1, 0, 0], + [1, 1, 0, 0], + [1, 1, 1, 0], + [1, 1, 1, 0]]) + mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="inclusive") + self.assertAllEqual(expected_mask, mask) + + +class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers, + test.TestCase): + + @property + def _autoregressive_flow_kwargs(self): + return { + "shift_and_log_scale_fn": masked_autoregressive_default_template( + hidden_layers=[2], shift_only=False), + "is_constant_jacobian": False, + } + + def testBijector(self): + x_ = np.arange(3 * 4 * 2).astype(np.float32).reshape(3, 4, 2) + with self.test_session() as sess: + ma = MaskedAutoregressiveFlow( + validate_args=True, + **self._autoregressive_flow_kwargs) + x = constant_op.constant(x_) + forward_x = ma.forward(x) + # Use identity to invalidate cache. + inverse_y = ma.inverse(array_ops.identity(forward_x)) + fldj = ma.forward_log_det_jacobian(x) + # Use identity to invalidate cache. + ildj = ma.inverse_log_det_jacobian(array_ops.identity(forward_x)) + variables.global_variables_initializer().run() + [ + forward_x_, + inverse_y_, + ildj_, + fldj_, + ] = sess.run([ + forward_x, + inverse_y, + ildj, + fldj, + ]) + self.assertEqual("masked_autoregressive_flow", ma.name) + self.assertAllClose(forward_x_, forward_x_, rtol=1e-6, atol=0.) + self.assertAllClose(x_, inverse_y_, rtol=1e-5, atol=0.) + self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.) + + def testMutuallyConsistent(self): + dims = 4 + with self.test_session() as sess: + ma = MaskedAutoregressiveFlow( + validate_args=True, + **self._autoregressive_flow_kwargs) + dist = transformed_distribution_lib.TransformedDistribution( + distribution=normal_lib.Normal(loc=0., scale=1.), + bijector=ma, + event_shape=[dims], + validate_args=True) + self.run_test_sample_consistent_log_prob( + sess=sess, + dist=dist, + num_samples=int(1e5), + radius=1., + center=0., + rtol=0.02) + + def testInvertMutuallyConsistent(self): + dims = 4 + with self.test_session() as sess: + ma = Invert(MaskedAutoregressiveFlow( + validate_args=True, + **self._autoregressive_flow_kwargs)) + dist = transformed_distribution_lib.TransformedDistribution( + distribution=normal_lib.Normal(loc=0., scale=1.), + bijector=ma, + event_shape=[dims], + validate_args=True) + self.run_test_sample_consistent_log_prob( + sess=sess, + dist=dist, + num_samples=int(1e5), + radius=1., + center=0., + rtol=0.02) + + +class MaskedAutoregressiveFlowShiftOnlyTest(MaskedAutoregressiveFlowTest): + + @property + def _autoregressive_flow_kwargs(self): + return { + "shift_and_log_scale_fn": masked_autoregressive_default_template( + hidden_layers=[2], shift_only=True), + "is_constant_jacobian": True, + } + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py index e62f900bbf..fd6c509446 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py @@ -26,6 +26,7 @@ @@Identity @@Inline @@Invert +@@MaskedAutoregressiveFlow @@Permute @@PowerTransform @@Sigmoid @@ -34,6 +35,9 @@ @@SoftmaxCentered @@Softplus @@Weibull + +@@masked_autoregressive_default_template +@@masked_dense """ from __future__ import absolute_import @@ -52,6 +56,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.exp import * from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import * from tensorflow.contrib.distributions.python.ops.bijectors.inline import * from tensorflow.contrib.distributions.python.ops.bijectors.invert import * +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import * from tensorflow.contrib.distributions.python.ops.bijectors.permute import * from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import * from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import * diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py new file mode 100644 index 0000000000..132dc570f9 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py @@ -0,0 +1,33 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MaskedAutoregressiveFlow bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive_impl import * +# pylint: enable=wildcard-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + "MaskedAutoregressiveFlow", + "masked_dense", + "masked_autoregressive_default_template", +] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py new file mode 100644 index 0000000000..ae14288393 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py @@ -0,0 +1,473 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MaskedAutoregressiveFlow bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.layers import core as layers +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import clip_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import template as template_ops +from tensorflow.python.ops import variable_scope as variable_scope_lib +from tensorflow.python.ops.distributions import bijector as bijector_lib + + +__all__ = [ + "MaskedAutoregressiveFlow", + "masked_autoregressive_default_template", + "masked_dense", +] + + +class MaskedAutoregressiveFlow(bijector_lib.Bijector): + """Affine MaskedAutoregressiveFlow bijector for vector-valued events. + + The affine autoregressive flow [1] provides a relatively simple framework for + user-specified (deep) architectures to learn a distribution over vector-valued + events. Regarding terminology, + + "Autoregressive models decompose the joint density as a product of + conditionals, and model each conditional in turn. Normalizing flows + transform a base density (e.g. a standard Gaussian) into the target density + by an invertible transformation with tractable Jacobian." [1] + + In other words, the "autoregressive property" is equivalent to the + decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided + `shift_and_log_scale_fn`, `masked_autoregressive_default_template`, achieves + this property by zeroing out weights in its `masked_dense` layers. + + In the `tf.distributions` framework, a "normalizing flow" is implemented as a + `tf.distributions.bijectors.Bijector`. The `forward` "autoregression" + is implemented using a `tf.while_loop` and a deep neural network (DNN) with + masked weights such that the autoregressive property is automatically met in + the `inverse`. + + A `TransformedDistribution` using `MaskedAutoregressiveFlow(...)` uses the + (expensive) forward-mode calculation to draw samples and the (cheap) + reverse-mode calculation to compute log-probabilities. Conversely, a + `TransformedDistribution` using `Invert(MaskedAutoregressiveFlow(...))` uses + the (expensive) forward-mode calculation to compute log-probabilities and the + (cheap) reverse-mode calculation to compute samples. See "Example Use" + [below] for more details. + + Given a `shift_and_log_scale_fn`, the forward and inverse transformations are + (a sequence of) affine transformations. A "valid" `shift_and_log_scale_fn` + must compute each `shift` (aka `loc` or "mu" [2]) and `log(scale)` (aka + "alpha" [2]) such that each are broadcastable with the arguments to `forward` + and `inverse`, i.e., such that the calculations in `forward`, `inverse` + [below] are possible. + + For convenience, `masked_autoregressive_default_template` is offered as a + possible `shift_and_log_scale_fn` function. It implements the MADE + architecture [2]. MADE is a feed-forward network that computes a `shift` and + `log(scale)` using `masked_dense` layers in a deep neural network. Weights are + masked to ensure the autoregressive property. It is possible that this + architecture is suboptimal for your task. To build alternative networks, + either change the arguments to `masked_autoregressive_default_template`, use + the `masked_dense` function to roll-out your own, or use some other + architecture, e.g., using `tf.layers`. + + Warning: no attempt is made to validate that the `shift_and_log_scale_fn` + enforces the "autoregressive property". + + Assuming `shift_and_log_scale_fn` has valid shape and autoregressive + semantics, the forward transformation is, + + ```python + def forward(x): + y = zeros_like(x) + event_size = x.shape[-1] + for _ in range(event_size): + shift, log_scale = shift_and_log_scale_fn(y) + y = x * math_ops.exp(log_scale) + shift + return y + ``` + + and the inverse transformation is, + + ```python + def inverse(y): + shift, log_scale = shift_and_log_scale_fn(y) + return (y - shift) / math_ops.exp(log_scale) + ``` + + Notice that the `inverse` does not need a for-loop. This is because in the + forward pass each calculation of `shift` and `log_scale` is based on the `y` + calculated so far (not `x`). In the `inverse`, the `y` is fully known, thus is + equivalent to the scaling used in `forward` after `event_size` passes, i.e., + the "last" `y` used to compute `shift`, `log_scale`. (Roughly speaking, this + also proves the transform is bijective.) + + #### Example Use + + ```python + ds = tf.contrib.distributions + bs = tf.contrib.distributions.bijectors + + dims = 5 + + # A common choice for a normalizing flow is to use a Gaussian for the base + # distribution. (However, any continuous distribution would work.) E.g., + maf = ds.TransformedDistribution( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.MaskedAutoregressiveFlow( + shift_and_log_scale_fn=bs.masked_autoregressive_default_template( + hidden_layers=[512, 512])), + event_shape=[dims]) + + x = maf.sample() # Expensive; uses `tf.while_loop`, no Bijector caching. + maf.log_prob(x) # Almost free; uses Bijector caching. + maf.log_prob(0.) # Cheap; no `tf.while_loop` despite no Bijector caching. + + # [1] also describes an "Inverse Autoregressive Flow", e.g., + iaf = ds.TransformedDistribution( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.Invert(bs.MaskedAutoregressiveFlow( + shift_and_log_scale_fn=bs.masked_autoregressive_default_template( + hidden_layers=[512, 512]))), + event_shape=[dims]) + + x = iaf.sample() # Cheap; no `tf.while_loop` despite no Bijector caching. + iaf.log_prob(x) # Almost free; uses Bijector caching. + iaf.log_prob(0.) # Expensive; uses `tf.while_loop`, no Bijector caching. + + # In many (if not most) cases the default `shift_and_log_scale_fn` will be a + # poor choice. Here's an example of using a "shift only" version and with a + # different number/depth of hidden layers. + shift_only = True + maf_no_scale_hidden2 = ds.TransformedDistribution( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.MaskedAutoregressiveFlow( + bs.masked_autoregressive_default_template( + hidden_layers=[32], + shift_only=shift_only), + is_constant_jacobian=shift_only), + event_shape=[dims]) + ``` + + [1]: "Masked Autoregressive Flow for Density Estimation." + George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017. + https://arxiv.org/abs/1705.07057 + + [2]: "MADE: Masked Autoencoder for Distribution Estimation." + Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015. + https://arxiv.org/abs/1502.03509 + + """ + + def __init__(self, + shift_and_log_scale_fn, + is_constant_jacobian=False, + validate_args=False, + name=None): + """Creates the MaskedAutoregressiveFlow bijector. + + Args: + shift_and_log_scale_fn: Python `callable` which computes `shift` and + `log_scale` from both the forward domain (`x`) and the inverse domain + (`y`). Calculation must respect the "autoregressive property" (see class + docstring). Suggested default + `masked_autoregressive_default_template(hidden_layers=...)`. + Typically the function contains `tf.Variables` and is wrapped using + `tf.make_template`. Returning `None` for either (both) `shift`, + `log_scale` is equivalent to (but more efficient than) returning zero. + is_constant_jacobian: Python `bool`. Default: `False`. When `True` the + implementation assumes `log_scale` does not depend on the forward domain + (`x`) or inverse domain (`y`) values. (No validation is made; + `is_constant_jacobian=False` is always safe but possibly computationally + inefficient.) + validate_args: Python `bool` indicating whether arguments should be + checked for correctness. + name: Python `str`, name given to ops managed by this object. + """ + name = name or "masked_autoregressive_flow" + self._shift_and_log_scale_fn = shift_and_log_scale_fn + super(MaskedAutoregressiveFlow, self).__init__( + is_constant_jacobian=is_constant_jacobian, + validate_args=validate_args, + name=name) + + def _forward(self, x): + event_size = array_ops.shape(x)[-1] + def _loop_body(index, y0): + """While-loop body for autoregression calculation.""" + # Set caching device to avoid re-getting the tf.Variable for every while + # loop iteration. + with variable_scope_lib.variable_scope( + variable_scope_lib.get_variable_scope()) as vs: + if vs.caching_device is None: + vs.set_caching_device(lambda op: op.device) + shift, log_scale = self._shift_and_log_scale_fn(y0) + y = x + if log_scale is not None: + y *= math_ops.exp(log_scale) + if shift is not None: + y += shift + return index + 1, y + _, y = control_flow_ops.while_loop( + cond=lambda index, _: index < event_size, + body=_loop_body, + loop_vars=[0, array_ops.zeros_like(x, name="y0")]) + return y + + def _inverse(self, y): + shift, log_scale = self._shift_and_log_scale_fn(y) + x = y + if shift is not None: + x -= shift + if log_scale is not None: + x *= math_ops.exp(-log_scale) + return x + + def _inverse_log_det_jacobian(self, y): + _, log_scale = self._shift_and_log_scale_fn(y) + if log_scale is None: + return constant_op.constant(0., dtype=y.dtype, name="ildj") + return -math_ops.reduce_sum(log_scale, axis=-1) + + +MASK_INCLUSIVE = "inclusive" +MASK_EXCLUSIVE = "exclusive" + + +def _gen_slices(num_blocks, n_in, n_out, mask_type=MASK_EXCLUSIVE): + """Generate the slices for building an autoregressive mask.""" + # TODO(b/67594795): Better support of dynamic shape. + slices = [] + col = 0 + d_in = n_in // num_blocks + d_out = n_out // num_blocks + row = d_out if mask_type == MASK_EXCLUSIVE else 0 + for _ in range(num_blocks): + row_slice = slice(row, None) + col_slice = slice(col, col + d_in) + slices.append([row_slice, col_slice]) + col += d_in + row += d_out + return slices + + +def _gen_mask(num_blocks, + n_in, + n_out, + mask_type=MASK_EXCLUSIVE, + dtype=dtypes.float32): + """Generate the mask for building an autoregressive dense layer.""" + # TODO(b/67594795): Better support of dynamic shape. + mask = np.zeros([n_out, n_in], dtype=dtype.as_numpy_dtype()) + slices = _gen_slices(num_blocks, n_in, n_out, mask_type=mask_type) + for [row_slice, col_slice] in slices: + mask[row_slice, col_slice] = 1 + return mask + + +def masked_dense(inputs, + units, + num_blocks=None, + exclusive=False, + kernel_initializer=None, + reuse=None, + name=None, + *args, + **kwargs): + """A autoregressively masked dense layer. Analogous to `tf.layers.dense`. + + See [1] for detailed explanation. + + [1]: "MADE: Masked Autoencoder for Distribution Estimation." + Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015. + https://arxiv.org/abs/1502.03509 + + Arguments: + inputs: Tensor input. + units: Python `int` scalar representing the dimensionality of the output + space. + num_blocks: Python `int` scalar representing the number of blocks for the + MADE masks. + exclusive: Python `bool` scalar representing whether to zero the diagonal of + the mask, used for the first layer of a MADE. + kernel_initializer: Initializer function for the weight matrix. + If `None` (default), weights are initialized using the + `tf.glorot_random_initializer`. + reuse: Python `bool` scalar representing whether to reuse the weights of a + previous layer by the same name. + name: Python `str` used to describe ops managed by this function. + *args: `tf.layers.dense` arguments. + **kwargs: `tf.layers.dense` keyword arguments. + + Returns: + Output tensor. + + Raises: + NotImplementedError: if rightmost dimension of `inputs` is unknown prior to + graph execution. + """ + # TODO(b/67594795): Better support of dynamic shape. + input_depth = inputs.shape.with_rank_at_least(1)[-1].value + if input_depth is None: + raise NotImplementedError( + "Rightmost dimension must be known prior to graph execution.") + + mask = _gen_mask(num_blocks, input_depth, units, + MASK_EXCLUSIVE if exclusive else MASK_INCLUSIVE).T + + if kernel_initializer is None: + kernel_initializer = init_ops.glorot_normal_initializer() + + def masked_initializer(shape, dtype=None, partition_info=None): + return mask * kernel_initializer(shape, dtype, partition_info) + + with ops.name_scope(name, "masked_dense", [inputs, units, num_blocks]): + layer = layers.Dense( + units, + kernel_initializer=masked_initializer, + kernel_constraint=lambda x: mask * x, + name=name, + dtype=inputs.dtype.base_dtype, + _scope=name, + _reuse=reuse, + *args, + **kwargs) + return layer.apply(inputs) + + +def masked_autoregressive_default_template( + hidden_layers, + shift_only=False, + activation=nn_ops.relu, + log_scale_min_clip=-5., + log_scale_max_clip=3., + log_scale_clip_gradient=False, + name=None, + *args, + **kwargs): + """Build the MADE Model [1]. + + This will be wrapped in a make_template to ensure the variables are only + created once. It takes the input and returns the `loc` ("mu" [1]) and + `log_scale` ("alpha" [1]) from the MADE network. + + Warning: This function uses `masked_dense` to create randomly initialized + `tf.Variables`. It is presumed that these will be fit, just as you would any + other neural architecture which uses `tf.layers.dense`. + + #### About Hidden Layers: + + Each element of `hidden_layers` should be greater than the `input_depth` + (i.e., `input_depth = tf.shape(input)[-1]` where `input` is the input to the + neural network). This is necessary to ensure the autoregressivity property. + + #### About Clipping: + + This function also optionally clips the `log_scale` (but possibly not its + gradient). This is useful because if `log_scale` is too small/large it might + underflow/overflow making it impossible for the `MaskedAutoregressiveFlow` + bijector to implement a bijection. Additionally, the `log_scale_clip_gradient` + `bool` indicates whether the gradient should also be clipped. The default does + not clip the gradient; this is useful because it still provides gradient + information (for fitting) yet solves the numerical stability problem. I.e., + `log_scale_clip_gradient = False` means + `grad[exp(clip(x))] = grad[x] exp(clip(x))` rather than the usual + `grad[clip(x)] exp(clip(x))`. + + [1]: "MADE: Masked Autoencoder for Distribution Estimation." + Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015. + https://arxiv.org/abs/1502.03509 + + Arguments: + hidden_layers: Python `list`-like of non-negative integer, scalars + indicating the number of units in each hidden layer. Default: `[512, 512]. + shift_only: Python `bool` indicating if only the `shift` term shall be + computed. Default: `False`. + activation: Activation function (callable). Explicitly setting to `None` + implies a linear activation. + log_scale_min_clip: `float`-like scalar `Tensor`, or a `Tensor` with the + same shape as `log_scale`. The minimum value to clip by. Default: -5. + log_scale_max_clip: `float`-like scalar `Tensor`, or a `Tensor` with the + same shape as `log_scale`. The maximum value to clip by. Default: 3. + log_scale_clip_gradient: Python `bool` indicating that the gradient of + `tf.clip_by_value` should be preserved. Default: `False`. + name: A name for ops managed by this function. Default: + "masked_autoregressive_default_template". + *args: `tf.layers.dense` arguments. + **kwargs: `tf.layers.dense` keyword arguments. + + Returns: + shift: `Float`-like `Tensor` of shift terms (the "mu" in [2]). + log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in [2]). + + Raises: + NotImplementedError: if rightmost dimension of `inputs` is unknown prior to + graph execution. + """ + + with ops.name_scope(name, "masked_autoregressive_default_template", + values=[log_scale_min_clip, log_scale_max_clip]): + def _fn(x): + """MADE parameterized via `masked_autoregressive_default_template`.""" + # TODO(b/67594795): Better support of dynamic shape. + input_depth = x.shape.with_rank_at_least(1)[-1].value + if input_depth is None: + raise NotImplementedError( + "Rightmost dimension must be known prior to graph execution.") + input_shape = (np.int32(x.shape.as_list()) if x.shape.is_fully_defined() + else array_ops.shape(x)) + for i, units in enumerate(hidden_layers): + x = masked_dense( + inputs=x, + units=units, + num_blocks=input_depth, + exclusive=True if i == 0 else False, + activation=activation, + *args, + **kwargs) + x = masked_dense( + inputs=x, + units=(1 if shift_only else 2) * input_depth, + num_blocks=input_depth, + activation=None, + *args, + **kwargs) + if shift_only: + x = array_ops.reshape(x, shape=input_shape) + return x, None + x = array_ops.reshape( + x, shape=array_ops.concat([input_shape, [2]], axis=0)) + shift, log_scale = array_ops.unstack(x, num=2, axis=-1) + which_clip = (math_ops.clip_by_value if log_scale_clip_gradient + else _clip_by_value_preserve_grad) + log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip) + return shift, log_scale + return template_ops.make_template( + "masked_autoregressive_default_template", _fn) + + +def _clip_by_value_preserve_grad(x, clip_value_min, clip_value_max, name=None): + """Clips input while leaving gradient unaltered.""" + with ops.name_scope(name, "clip_by_value_preserve_grad", + [x, clip_value_min, clip_value_max]): + clip_x = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) + return x + array_ops.stop_gradient(clip_x - x) diff --git a/tensorflow/contrib/distributions/python/ops/test_util.py b/tensorflow/contrib/distributions/python/ops/test_util.py index da7d3907ac..631ffc1bac 100644 --- a/tensorflow/contrib/distributions/python/ops/test_util.py +++ b/tensorflow/contrib/distributions/python/ops/test_util.py @@ -25,6 +25,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import histogram_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import variables as variables_ops __all__ = [ @@ -279,26 +280,32 @@ class VectorDistributionTestHelpers(object): def monte_carlo_hypersphere_volume(dist, num_samples, radius, center): # https://en.wikipedia.org/wiki/Importance_sampling x = dist.sample(num_samples, seed=seed) + x = array_ops.identity(x) # Invalidate bijector cacheing. return math_ops.reduce_mean( math_ops.exp(-dist.log_prob(x)) * is_in_ball(x, radius, center), axis=0) - [ - batch_shape_, - actual_volume_, - sample_volume_, - ] = sess.run([ - dist.batch_shape_tensor(), - actual_hypersphere_volume( - dims=dist.event_shape_tensor()[0], - radius=radius), - monte_carlo_hypersphere_volume( - dist, - num_samples=num_samples, - radius=radius, - center=center), - ]) - + # Build graph. + with ops.name_scope( + "run_test_sample_consistent_log_prob", + values=[num_samples, radius, center] + dist._graph_parents): # pylint: disable=protected-access + batch_shape = dist.batch_shape_tensor() + actual_volume = actual_hypersphere_volume( + dims=dist.event_shape_tensor()[0], + radius=radius) + sample_volume = monte_carlo_hypersphere_volume( + dist, + num_samples=num_samples, + radius=radius, + center=center) + init_op = variables_ops.global_variables_initializer() + + # Execute graph. + sess.run(init_op) + [batch_shape_, actual_volume_, sample_volume_] = sess.run([ + batch_shape, actual_volume, sample_volume]) + + # Check results. self.assertAllClose(np.tile(actual_volume_, reps=batch_shape_), sample_volume_, rtol=rtol, atol=atol) -- GitLab From 251070eb2d7dc8376c868b8a86342c3332e706f0 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Wed, 18 Oct 2017 15:17:14 -0700 Subject: [PATCH 126/573] Implement nest.flatten in C++ This function is used extensively in imperative_grad. Implementing it in C++ reduces SPINN training time by over 8%. PiperOrigin-RevId: 172663591 --- tensorflow/contrib/cmake/tf_python.cmake | 2 + tensorflow/python/BUILD | 13 +++ tensorflow/python/tensorflow.i | 2 + tensorflow/python/util/nest.py | 41 ++------ tensorflow/python/util/util.cc | 127 +++++++++++++++++++++++ tensorflow/python/util/util.h | 74 +++++++++++++ tensorflow/python/util/util.i | 42 ++++++++ 7 files changed, 269 insertions(+), 32 deletions(-) create mode 100644 tensorflow/python/util/util.cc create mode 100644 tensorflow/python/util/util.h create mode 100644 tensorflow/python/util/util.i diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index e83618a94e..8ddfb59595 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -874,6 +874,8 @@ set (pywrap_tensorflow_internal_src "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.cc" "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.h" "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.cc" + "${tensorflow_source_dir}/tensorflow/python/util/util.h" + "${tensorflow_source_dir}/tensorflow/python/util/util.cc" "${tensorflow_source_dir}/tensorflow/cc/framework/ops.cc" "${tensorflow_source_dir}/tensorflow/cc/framework/scope.cc" "${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.cc" diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index cbeb0b46cb..21cdaec477 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -245,6 +245,17 @@ cc_library( ], ) +cc_library( + name = "cpp_python_util", + srcs = ["util/util.cc"], + hdrs = ["util/util.h"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//util/python:python_headers", + ], +) + cc_library( name = "py_func_lib", srcs = ["lib/core/py_func.cc"], @@ -2982,10 +2993,12 @@ tf_py_wrap_cc( "util/stat_summarizer.i", "util/tfprof.i", "util/transform_graph.i", + "util/util.i", ], deps = [ ":cost_analyzer_lib", ":model_analyzer_lib", + ":cpp_python_util", ":cpp_shape_inference", ":kernel_registry", ":numpy_lib", diff --git a/tensorflow/python/tensorflow.i b/tensorflow/python/tensorflow.i index 9cef765bf3..d221dd523b 100644 --- a/tensorflow/python/tensorflow.i +++ b/tensorflow/python/tensorflow.i @@ -44,6 +44,8 @@ limitations under the License. %include "tensorflow/python/util/transform_graph.i" +%include "tensorflow/python/util/util.i" + %include "tensorflow/python/grappler/cluster.i" %include "tensorflow/python/grappler/item.i" %include "tensorflow/python/grappler/tf_optimizer.i" diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py index d57140da75..dd6acee3c7 100644 --- a/tensorflow/python/util/nest.py +++ b/tensorflow/python/util/nest.py @@ -35,7 +35,7 @@ import collections as _collections import six as _six -from tensorflow.python.platform import tf_logging as _tf_logging +from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow from tensorflow.python.util.all_util import remove_undocumented @@ -91,26 +91,6 @@ def _yield_value(iterable): yield value -def _yield_flat_nest(nest): - for n in _yield_value(nest): - if is_sequence(n): - for ni in _yield_flat_nest(n): - yield ni - else: - yield n - - -# Used by `_warn_once` to remember which warning messages have been given. -_ALREADY_WARNED = {} - - -def _warn_once(message): - """Logs a warning message, once per unique string.""" - if message not in _ALREADY_WARNED: - _ALREADY_WARNED[message] = True - _tf_logging.warning(message) - - def is_sequence(seq): """Returns a true if its input is a collections.Sequence (except strings). @@ -121,13 +101,7 @@ def is_sequence(seq): True if the sequence is a not a string and is a collections.Sequence or a dict. """ - if isinstance(seq, dict): - return True - if isinstance(seq, set): - _warn_once("Sets are not currently considered sequences, but this may " - "change in the future, so consider avoiding using them.") - return (isinstance(seq, _collections.Sequence) - and not isinstance(seq, _six.string_types)) + return _pywrap_tensorflow.IsSequence(seq) def flatten(nest): @@ -145,6 +119,9 @@ def flatten(nest): a correponding plain dict, or vice-versa. Dictionaries with non-sortable keys cannot be flattened. + Users must not modify any collections used in `nest` while this function is + running. + Args: nest: an arbitrarily nested structure or a scalar object. Note, numpy arrays are considered scalars. @@ -155,10 +132,7 @@ def flatten(nest): Raises: TypeError: The nest is or contains a dict with non-sortable keys. """ - if is_sequence(nest): - return list(_yield_flat_nest(nest)) - else: - return [nest] + return _pywrap_tensorflow.Flatten(nest) def _recursive_assert_same_structure(nest1, nest2, check_types): @@ -692,6 +666,9 @@ def get_traverse_shallow_structure(traverse_fn, structure): return _sequence_like(structure, level_traverse) +_pywrap_tensorflow.RegisterSequenceClass(_collections.Sequence) + + _allowed_symbols = [ "assert_same_structure", "is_sequence", diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc new file mode 100644 index 0000000000..c3d7611ad4 --- /dev/null +++ b/tensorflow/python/util/util.cc @@ -0,0 +1,127 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/python/util/util.h" + +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { +namespace swig { + +namespace { + +// Type object for collections.Sequence. This is set by RegisterSequenceClass. +PyObject* CollectionsSequenceType = nullptr; + +bool WarnedThatSetIsNotSequence = false; + +// Returns 1 if `o` is considered a sequence for the purposes of Flatten(). +// Returns 0 otherwise. +// Returns -1 if an error occured. +int IsSequenceHelper(PyObject* o) { + if (PyDict_Check(o)) return true; + if (PySet_Check(o) && !WarnedThatSetIsNotSequence) { + LOG(WARNING) << "Sets are not currently considered sequences, " + "but this may change in the future, " + "so consider avoiding using them."; + WarnedThatSetIsNotSequence = true; + } + if (CollectionsSequenceType == nullptr) { + PyErr_SetString( + PyExc_RuntimeError, + tensorflow::strings::StrCat( + "collections.Sequence type has not been set. " + "Please call RegisterSequenceClass before using this module") + .c_str()); + return -1; + } + int is_instance = PyObject_IsInstance(o, CollectionsSequenceType); + if (is_instance == -1) return -1; + return static_cast(is_instance != 0 && !PyBytes_Check(o) && +#if PY_MAJOR_VERSION < 3 + !PyString_Check(o) && +#endif + !PyUnicode_Check(o)); +} + +bool FlattenHelper(PyObject* nested, PyObject* list) { + // if nested is not a sequence, append itself and exit + int is_seq = IsSequenceHelper(nested); + if (is_seq == -1) return false; + if (!is_seq) { + return PyList_Append(list, nested) != -1; + } + + // if nested if dictionary, sort it by key and recurse on each value + if (PyDict_Check(nested)) { + PyObject* keys = PyDict_Keys(nested); + if (PyList_Sort(keys) == -1) return false; + Py_ssize_t size = PyList_Size(keys); + for (Py_ssize_t i = 0; i < size; ++i) { + // We know that key and val will not be deleted because nested owns + // a reference to them and callers of flatten must not modify nested + // while the method is running. + PyObject* key = PyList_GET_ITEM(keys, i); + PyObject* val = PyDict_GetItem(nested, key); + if (Py_EnterRecursiveCall(" in Flatten")) { + Py_DECREF(keys); + return false; + } + FlattenHelper(val, list); + Py_LeaveRecursiveCall(); + } + Py_DECREF(keys); + return true; + } + + // iterate and recurse + PyObject* item; + PyObject* iterator = PyObject_GetIter(nested); + while ((item = PyIter_Next(iterator)) != nullptr) { + FlattenHelper(item, list); + Py_DECREF(item); + } + Py_DECREF(iterator); + return true; +} + +} // anonymous namespace + +void RegisterSequenceClass(PyObject* sequence_class) { + if (!PyType_Check(sequence_class)) { + PyErr_SetString( + PyExc_TypeError, + tensorflow::strings::StrCat( + "Expecting a class definition for `collections.Sequence`. Got ", + Py_TYPE(sequence_class)->tp_name) + .c_str()); + return; + } + CollectionsSequenceType = sequence_class; +} + +bool IsSequence(PyObject* o) { return IsSequenceHelper(o) == 1; } + +PyObject* Flatten(PyObject* nested) { + PyObject* list = PyList_New(0); + if (FlattenHelper(nested, list)) { + return list; + } else { + Py_DECREF(list); + return nullptr; + } +} +} // namespace swig +} // namespace tensorflow diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h new file mode 100644 index 0000000000..493d26b497 --- /dev/null +++ b/tensorflow/python/util/util.h @@ -0,0 +1,74 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Functions for getting information about kernels registered in the binary. +#ifndef THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_ +#define THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_ + +#include + +namespace tensorflow { +namespace swig { + +// Implements the same interface as tensorflow.util.nest.is_sequence +// Returns a true if its input is a collections.Sequence (except strings). +// +// Args: +// seq: an input sequence. +// +// Returns: +// True if the sequence is a not a string and is a collections.Sequence or a +// dict. +bool IsSequence(PyObject* o); + +// Implements the same interface as tensorflow.util.nest.flatten +// +// Returns a flat list from a given nested structure. +// +// If `nest` is not a sequence, tuple, or dict, then returns a single-element +// list: `[nest]`. +// +// In the case of dict instances, the sequence consists of the values, sorted by +// key to ensure deterministic behavior. This is true also for `OrderedDict` +// instances: their sequence order is ignored, the sorting order of keys is +// used instead. The same convention is followed in `pack_sequence_as`. This +// correctly repacks dicts and `OrderedDict`s after they have been flattened, +// and also allows flattening an `OrderedDict` and then repacking it back using +// a correponding plain dict, or vice-versa. +// Dictionaries with non-sortable keys cannot be flattened. +// +// Args: +// nest: an arbitrarily nested structure or a scalar object. Note, numpy +// arrays are considered scalars. +// +// Returns: +// A Python list, the flattened version of the input. +// On error, returns nullptr +// +// Raises: +// TypeError: The nest is or contains a dict with non-sortable keys. +PyObject* Flatten(PyObject* nested); + +// RegisterSequenceClass is used to pass PyTypeObject for collections.Sequence +// (which is defined in python) into the C++ world. +// Alternative approach could be to import the collections modules and retrieve +// the type from the module. This approach also requires some trigger from +// Python so that we know that Python interpreter had been initialzied. +void RegisterSequenceClass(PyObject* sequence_class); + +} // namespace swig +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_ diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i new file mode 100644 index 0000000000..d69084fc00 --- /dev/null +++ b/tensorflow/python/util/util.i @@ -0,0 +1,42 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +%include "tensorflow/python/platform/base.i" + +%{ +#include "tensorflow/python/util/util.h" +%} + +%ignoreall + +%unignore tensorflow; +%unignore tensorflow::swig; +// The %exception block defined in tf_session.i releases the Python GIL for +// the length of each wrapped method. This file is included in tensorflow.i +// after tf_session.i and inherits this definition. We disable this behavior +// for functions in this module because they use python methods that need GIL. +// TODO(iga): Find a way not to leak such definitions across files. + +%unignore tensorflow::swig::RegisterSequenceClass; +%noexception tensorflow::swig::RegisterSequenceClass; + +%unignore tensorflow::swig::IsSequence; +%noexception tensorflow::swig::IsSequence; + +%unignore tensorflow::swig::Flatten; +%noexception tensorflow::swig::Flatten; + +%include "tensorflow/python/util/util.h" + +%unignoreall -- GitLab From 9b3233d25e0dd7078667712f128657f3fbb7dbd4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 15:53:02 -0700 Subject: [PATCH 127/573] Use Windows compatible string comparisons for setting cuda device flags. PiperOrigin-RevId: 172669121 --- tensorflow/stream_executor/cuda/cuda_platform.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_platform.cc b/tensorflow/stream_executor/cuda/cuda_platform.cc index d69953f557..874ac1ab65 100644 --- a/tensorflow/stream_executor/cuda/cuda_platform.cc +++ b/tensorflow/stream_executor/cuda/cuda_platform.cc @@ -45,12 +45,11 @@ const DeviceOptions GetDeviceOptionsFromEnv() { } unsigned device_flags = 0; - if (strcasecmp(kScheduleSpinString, gpu_schedule_string) == 0) { + if (strcmp(kScheduleSpinString, gpu_schedule_string) == 0) { device_flags = perftools::gputools::DeviceOptions::kScheduleSpin; - } else if (strcasecmp(kScheduleYieldString, gpu_schedule_string) == 0) { + } else if (strcmp(kScheduleYieldString, gpu_schedule_string) == 0) { device_flags = perftools::gputools::DeviceOptions::kScheduleYield; - } else if (strcasecmp(kScheduleBlockingSyncString, gpu_schedule_string) == - 0) { + } else if (strcmp(kScheduleBlockingSyncString, gpu_schedule_string) == 0) { device_flags = perftools::gputools::DeviceOptions::kScheduleBlockingSync; } else { LOG(QFATAL) << "Unknown option for environment variable " -- GitLab From d94db4be7ebc02c2de169274f9f863611e68f98d Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Wed, 18 Oct 2017 16:10:42 -0700 Subject: [PATCH 128/573] Branch 172647355 (#13819) * Update inception score to match the openAI version from https://github.com/openai/improved-gan/tree/master/inception_score. PiperOrigin-RevId: 172562573 * Stub support for retrieving LossFunction by name. PiperOrigin-RevId: 172588516 * Don't emit fusion computations separately in HloModule::ToString. These computations are emitted with their fusion instruction and therefore don't need to be emitted as a separate comptutation in the module. PiperOrigin-RevId: 172612725 * Make `tf.contrib.distributions` quadrature family parameterized by `quadrature_grid_and_prob` vs `quadrature_degree`. Enables support of quadrature methods other than Gauss-Hermite. PiperOrigin-RevId: 172622919 * Fixes test breakage. PiperOrigin-RevId: 172626499 * Remove global step read dependency from model_fn. Estimator behavior still will be deterministic since the step checking logic in session_run_hooks was changed as follows: * assume stale step * before using the step, check for the current value by session.run PiperOrigin-RevId: 172629797 * More changs to avoid flakes in random_shuffle_queue_test PiperOrigin-RevId: 172630989 * Add expected keys to predictor exception if unexpected key detected. PiperOrigin-RevId: 172634275 * Add TF_GraphGetOpDef() to C API and use in Operation.op_def() Note that this creates a small change in behavior with the C API enabled, since previously not all Python Operations had an OpDef (op_def() returns None). With the C API enabled, op_def() always returns an OpDef. PiperOrigin-RevId: 172634411 * Implement ZlibInputStream::Tell() by keeping track of the number of bytes consumed by the reader. PiperOrigin-RevId: 172634455 * Upgrade tensorflow pip dependency version to 3.4.0+ PiperOrigin-RevId: 172635727 * [XLA] Deterministically dump an executable. Previously, dumping a executable is nondeterministic as a map in protobuf is serialized in random order. This CL enables "Deterministic dump" mode of protobuf, which sorts the map first before dumping them. This is helpful in comparing if two dumps are the same in XLA determinism test. PiperOrigin-RevId: 172637100 * Fixed work size computation in Split and SplitV ops to avoid integer overflow. PiperOrigin-RevId: 172637818 * Internal change. PiperOrigin-RevId: 172641543 * Bug fixes for fold_constants_lib. 1. Tensor names in TF may be in the form of "a:0", "a:1", or "a" as a shorthand notation of "a:0". FoldConstant library always expected the shorthand notation, and did not handle the cases where explicit notation was passed to input or output list. This means that this library could not handle the case when input or output were not the first output of a node. 2. To match the input nodes in the original graph and the added Recv nodes in rewritten graph, FoldConstant library used prefix matching. Unfortunately, this means that when a input name is a prefix of another input name, there is possibility that wrong Recv node gets matched. For example, if input names were "placeholder" and "placeholder_1", then it did not handle the case very well. 3. RemoveUnusedNodes() in FoldConstants lib could remove nodes which output depended on. This happened when an input name points to a node with multiple outputs and not all outputs of that node were included in the input names. 4. ReplaceSendRecvs() in FoldConstants lib assumed that all input nodes are removed during rewriting the graph. This assumption is not necessarily true, and it could add a duplicate node in the graph. PiperOrigin-RevId: 172641947 * Adds visibility to sgdr_learning_rate_decay. Currently SGD with warm restarts is siloed in tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py, since it is not listed in the 'training_py' build filegroup. This change simply adds sgdr_learning_rate_decay to this filegroup so that other projects can use warm restarts during optimization. PiperOrigin-RevId: 172643218 * Add logging verbosity to mnist.py PiperOrigin-RevId: 172643922 * Automated g4 rollback of changelist 172336111 PiperOrigin-RevId: 172645893 * Correct the docstring to reflect that the values of cols_to_vars are always lists of Variable's (never single Variable's or PartitionedVariables), and make this true for bias. PiperOrigin-RevId: 172646456 * Changes MultiLabelHead.create_loss to return a Tensor of size [batch_size, 1], to be consistent with other heads. PiperOrigin-RevId: 172647355 * Disabling failing contrib tests. --- tensorflow/c/c_api.cc | 11 + tensorflow/c/c_api.h | 7 + tensorflow/c/c_api_function_test.cc | 21 ++ tensorflow/c/c_api_test.cc | 31 +++ tensorflow/compiler/xla/BUILD | 1 + tensorflow/compiler/xla/service/BUILD | 2 + tensorflow/compiler/xla/service/executable.cc | 8 +- tensorflow/compiler/xla/service/hlo_module.cc | 4 +- tensorflow/compiler/xla/util.h | 1 + .../contrib/data/python/kernel_tests/BUILD | 6 + .../kernel_tests/poisson_lognormal_test.py | 20 +- .../python/ops/poisson_lognormal.py | 54 +++-- .../python/ops/vector_diffeomixture.py | 55 +++-- .../estimator/python/estimator/head.py | 16 +- .../estimator/python/estimator/head_test.py | 32 +-- .../eval/python/classifier_metrics_impl.py | 81 ++++--- .../eval/python/classifier_metrics_test.py | 10 +- .../kernel_tests/layer_collection_test.py | 67 ++++++ .../python/kernel_tests/optimizer_test.py | 6 +- .../kfac/python/ops/layer_collection.py | 58 ++++- .../python/learn/estimators/estimator.py | 5 +- tensorflow/contrib/predictor/predictor.py | 6 +- tensorflow/contrib/training/BUILD | 1 + tensorflow/core/kernels/split_op.cc | 8 +- tensorflow/core/kernels/split_v_op.cc | 8 +- tensorflow/core/lib/io/zlib_buffers_test.cc | 172 ++++++++++++--- tensorflow/core/lib/io/zlib_inputstream.cc | 8 +- tensorflow/core/lib/io/zlib_inputstream.h | 3 + tensorflow/examples/learn/mnist.py | 3 + tensorflow/python/BUILD | 5 +- tensorflow/python/eager/BUILD | 7 +- tensorflow/python/eager/ops_test.py | 6 +- tensorflow/python/estimator/estimator.py | 7 +- tensorflow/python/feature_column/BUILD | 1 + .../python/feature_column/feature_column.py | 33 +-- .../feature_column/feature_column_test.py | 61 +++++- tensorflow/python/framework/ops.py | 26 ++- tensorflow/python/framework/ops_test.py | 15 ++ .../kernel_tests/random_shuffle_queue_test.py | 19 +- .../resource_variable_ops_test.py | 10 + .../python/ops/resource_variable_ops.py | 33 +++ tensorflow/python/training/adam_test.py | 81 +++---- tensorflow/python/training/saver_test.py | 47 ++-- .../graph_transforms/fold_constants_lib.cc | 202 ++++++++---------- .../graph_transforms/fold_constants_test.cc | 85 +++++++- tensorflow/tools/pip_package/setup.py | 2 +- 46 files changed, 951 insertions(+), 394 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 79fbd8c90c..cd98393e0a 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -1799,6 +1799,17 @@ void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def, status->status = MessageToBuffer(def, output_graph_def); } +void TF_GraphGetOpDef(TF_Graph* graph, const char* op_name, + TF_Buffer* output_op_def, TF_Status* status) { + const OpDef* op_def; + { + mutex_lock l(graph->mu); + status->status = graph->graph.op_registry()->LookUpOpDef(op_name, &op_def); + if (!status->status.ok()) return; + } + status->status = MessageToBuffer(*op_def, output_op_def); +} + TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions() { return new TF_ImportGraphDefOptions; } diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index 76cfcd5e0d..1e8bfdc7b0 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -864,6 +864,13 @@ TF_CAPI_EXPORT extern void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def, TF_Status* status); +// Returns the serialized OpDef proto with name `op_name`, or a bad status if no +// such op exists. This can return OpDefs of functions copied into the graph. +TF_CAPI_EXPORT extern void TF_GraphGetOpDef(TF_Graph* graph, + const char* op_name, + TF_Buffer* output_op_def, + TF_Status* status); + // TF_ImportGraphDefOptions holds options that can be passed to // TF_GraphImportGraphDef. typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions; diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc index 4db9a90fdc..d5580b6589 100644 --- a/tensorflow/c/c_api_function_test.cc +++ b/tensorflow/c/c_api_function_test.cc @@ -1465,5 +1465,26 @@ TEST_F(CApiFunctionTest, AppendHash) { ASSERT_EQ(string("func_name_base_qaJ8jA8UmGY"), fdef.signature().name()); } +TEST_F(CApiFunctionTest, GetOpDef) { + DefineFunction(func_name_, &func_); + TF_GraphCopyFunction(host_graph_, func_, nullptr, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Test we can retrieve function OpDef from graph + TF_Buffer* buffer = TF_NewBuffer(); + TF_GraphGetOpDef(host_graph_, func_name_, buffer, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Sanity check returned OpDef + string data(static_cast(buffer->data), buffer->length); + OpDef op_def; + op_def.ParseFromString(data); + EXPECT_EQ(op_def.name(), func_name_); + EXPECT_EQ(op_def.input_arg_size(), 1); + EXPECT_EQ(op_def.output_arg_size(), 1); + + TF_DeleteBuffer(buffer); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc index c442029009..d220bc5e95 100644 --- a/tensorflow/c/c_api_test.cc +++ b/tensorflow/c/c_api_test.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/graph/tensor_id.h" #include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -50,6 +51,11 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst); namespace { +static void ExpectHasSubstr(StringPiece s, StringPiece expected) { + EXPECT_TRUE(StringPiece(s).contains(expected)) + << "'" << s << "' does not contain '" << expected << "'"; +} + TEST(CAPI, Version) { EXPECT_STRNE("", TF_Version()); } TEST(CAPI, Status) { @@ -837,6 +843,31 @@ TEST(CAPI, ShapeInferenceError) { TF_DeleteStatus(status); } +TEST(CAPI, GetOpDef) { + TF_Status* status = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + TF_Buffer* buffer = TF_NewBuffer(); + + TF_GraphGetOpDef(graph, "Add", buffer, status); + ASSERT_EQ(TF_OK, TF_GetCode(status)); + const OpDef* expected_op_def; + TF_ASSERT_OK(OpRegistry::Global()->LookUpOpDef("Add", &expected_op_def)); + string expected_serialized; + expected_op_def->SerializeToString(&expected_serialized); + string actual_string(reinterpret_cast(buffer->data), + buffer->length); + EXPECT_EQ(expected_serialized, actual_string); + + TF_GraphGetOpDef(graph, "MyFakeOp", buffer, status); + EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status)); + ExpectHasSubstr(TF_Message(status), + "Op type not registered 'MyFakeOp' in binary"); + + TF_DeleteBuffer(buffer); + TF_DeleteGraph(graph); + TF_DeleteStatus(status); +} + void StringVectorToArrays(const std::vector& v, std::unique_ptr* ptrs, std::unique_ptr* lens) { diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index be87506d3c..e51bbffcd0 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -171,6 +171,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":status", + ":status_macros", ":types", ":xla_data_proto", "//tensorflow/core:lib", diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index ed42358e7e..b3fbb0c513 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -579,12 +579,14 @@ cc_library( ":shaped_buffer", ":versioned_computation_handle", "//tensorflow/compiler/xla:executable_run_options", + "//tensorflow/compiler/xla:status", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/legacy_flags:debug_options_flags", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/stream_executor", ], diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index 62b8fa6a2b..9c96d9eb30 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -17,7 +17,9 @@ limitations under the License. #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h" #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h" +#include "tensorflow/compiler/xla/status.h" #include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/env.h" @@ -82,7 +84,11 @@ Status Executable::DumpSessionModule() { } filename = SanitizeFileName(std::move(filename)); string file_path = tensorflow::io::JoinPath(directory_path, filename); - return tensorflow::WriteBinaryProto(env, file_path, session_module); + string result; + TF_RET_CHECK( + tensorflow::SerializeToStringDeterministic(session_module, &result)); + return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path, + result); } } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 5bc7a36439..9d4a994838 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -154,8 +154,8 @@ string HloModule::ToString() const { std::ostringstream s; s << "HloModule " << name() << ":\n\n"; s << "ENTRY " << entry_computation()->ToString() << "\n\n"; - for (const std::unique_ptr& computation : computations_) { - if (computation.get() != entry_computation()) { + for (const HloComputation* computation : MakeNonfusionComputations()) { + if (computation != entry_computation()) { s << computation->ToString() << "\n\n"; } } diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h index f6c0bd1563..f58f57b443 100644 --- a/tensorflow/compiler/xla/util.h +++ b/tensorflow/compiler/xla/util.h @@ -24,6 +24,7 @@ limitations under the License. #include #include "tensorflow/compiler/xla/status.h" +#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/core/status.h" diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index c34c9dad9b..7ec049d29b 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -11,6 +11,9 @@ py_test( size = "small", srcs = ["batch_dataset_op_test.py"], srcs_version = "PY2AND3", + tags = [ + "manual", # b/67958604 + ], deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", @@ -352,6 +355,9 @@ py_test( size = "small", srcs = ["sloppy_transformation_dataset_op_test.py"], srcs_version = "PY2AND3", + tags = [ + "manual", # b/67958761 + ], deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py index 7cb46bb236..3ded4159d8 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import numpy as np + from tensorflow.contrib.distributions.python.ops import poisson_lognormal from tensorflow.contrib.distributions.python.ops import test_util from tensorflow.python.platform import test @@ -32,7 +34,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=-2., scale=1.1, - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( sess, pln, rtol=0.1) @@ -42,7 +45,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=0., scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( sess, pln, rtol=0.02) @@ -52,7 +56,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( sess, pln, rtol=0.1, atol=0.01) @@ -62,7 +67,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( sess, pln, rtol=0.1, atol=0.01) @@ -72,7 +78,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[[0.], [-0.5]], scale=[[1., 0.9]], - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( sess, pln, rtol=0.1, atol=0.08) @@ -82,7 +89,8 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[[0.], [-0.5]], scale=[[1., 0.9]], - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( sess, pln, rtol=0.1, atol=0.01) diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 65ee3a16d6..80d4e2dc5e 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -93,7 +93,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): : d=0, ..., deg-1 } ``` - where, [`grid, w = numpy.polynomial.hermite.hermgauss(deg)`]( + where, [e.g., `grid, w = numpy.polynomial.hermite.hermgauss(deg)`]( https://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.polynomial.hermite.hermgauss.html) and `prob = w / sqrt(pi)`. @@ -106,14 +106,15 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): pln = ds.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) """ def __init__(self, loc, scale, - quadrature_polynomial_degree=8, + quadrature_grid_and_probs=None, validate_args=False, allow_nan_stats=True, name="PoissonLogNormalQuadratureCompound"): @@ -124,8 +125,9 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): the LogNormal prior. scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of the LogNormal prior. - quadrature_polynomial_degree: Python `int`-like scalar. - Default value: 8. + quadrature_grid_and_probs: Python pair of `list`-like objects representing + the sample points and the corresponding (possibly normalized) weight. + When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -138,6 +140,8 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): Raises: TypeError: if `loc.dtype != scale[0].dtype`. + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` """ parameters = locals() with ops.name_scope(name, values=[loc, scale]): @@ -153,18 +157,21 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format( loc.dtype.name, scale.dtype.name)) - self._degree = quadrature_polynomial_degree - - grid, prob = np.polynomial.hermite.hermgauss( - deg=quadrature_polynomial_degree) - - # It should be that `sum(prob) == sqrt(pi)`, but self-normalization is - # more numerically stable. - prob = prob.astype(dtype.as_numpy_dtype) - prob /= np.linalg.norm(prob, ord=1) + if quadrature_grid_and_probs is None: + grid, probs = np.polynomial.hermite.hermgauss(deg=8) + else: + grid, probs = tuple(quadrature_grid_and_probs) + if len(grid) != len(probs): + raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length list-like objects") + grid = grid.astype(dtype.as_numpy_dtype) + probs = probs.astype(dtype.as_numpy_dtype) + probs /= np.linalg.norm(probs, ord=1) + self._quadrature_grid = grid + self._quadrature_probs = probs self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(prob), + logits=np.log(probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -210,9 +217,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): return self._scale @property - def quadrature_polynomial_degree(self): - """Polynomial largest exponent used for Gauss-Hermite quadrature.""" - return self._degree + def quadrature_grid(self): + """Quadrature grid points.""" + return self._quadrature_grid + + @property + def quadrature_probs(self): + """Quadrature normalized weights.""" + return self._quadrature_probs def _batch_shape_tensor(self): return array_ops.broadcast_dynamic_shape( @@ -242,10 +254,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): [batch_size])), seed=distribution_util.gen_new_seed( seed, "poisson_lognormal_quadrature_compound")) - # Stride `quadrature_polynomial_degree` for `batch_size` number of times. + # Stride `quadrature_degree` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * self._degree, - delta=self._degree, + limit=batch_size * len(self.quadrature_probs), + delta=len(self.quadrature_probs), dtype=ids.dtype) ids += offset rate = array_ops.gather( diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py index 438d628da4..33dad811a9 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py @@ -141,7 +141,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): and, ```none - grid, weight = np.polynomial.hermite.hermgauss(quadrature_polynomial_degree) + grid, weight = np.polynomial.hermite.hermgauss(quadrature_degree) prob[k] = weight[k] / sqrt(pi) lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i]) ``` @@ -219,7 +219,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): distribution, loc=None, scale=None, - quadrature_polynomial_degree=8, + quadrature_grid_and_probs=None, validate_args=False, allow_nan_stats=True, name="VectorDiffeomixture"): @@ -248,7 +248,9 @@ class VectorDiffeomixture(distribution_lib.Distribution): `k`-th element represents the `scale` used for the `k`-th affine transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`, `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices - quadrature_polynomial_degree: Python `int`-like scalar. + quadrature_grid_and_probs: Python pair of `list`-like objects representing + the sample points and the corresponding (possibly normalized) weight. + When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -262,7 +264,8 @@ class VectorDiffeomixture(distribution_lib.Distribution): Raises: ValueError: if `not scale or len(scale) < 2`. ValueError: if `len(loc) != len(scale)` - ValueError: if `quadrature_polynomial_degree < 1`. + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` ValueError: if `validate_args` and any not scale.is_positive_definite. TypeError: if any scale.dtype != scale[0].dtype. TypeError: if any loc.dtype != scale[0].dtype. @@ -307,12 +310,6 @@ class VectorDiffeomixture(distribution_lib.Distribution): name="endpoint_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip(loc, scale))] - if quadrature_polynomial_degree < 1: - raise ValueError("quadrature_polynomial_degree={} " - "is not at least 1".format( - quadrature_polynomial_degree)) - self._degree = quadrature_polynomial_degree - # TODO(jvdillon): Remove once we support k-mixtures. # We make this assertion here because otherwise `grid` would need to be a # vector not a scalar. @@ -320,17 +317,24 @@ class VectorDiffeomixture(distribution_lib.Distribution): raise NotImplementedError("Currently only bimixtures are supported; " "len(scale)={} is not 2.".format(len(scale))) - grid, prob = np.polynomial.hermite.hermgauss( - deg=quadrature_polynomial_degree) + if quadrature_grid_and_probs is None: + grid, probs = np.polynomial.hermite.hermgauss(deg=8) + else: + grid, probs = tuple(quadrature_grid_and_probs) + if len(grid) != len(probs): + raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length list-like objects") grid = grid.astype(dtype.as_numpy_dtype) - prob = prob.astype(dtype.as_numpy_dtype) - prob /= np.linalg.norm(prob, ord=1) + probs = probs.astype(dtype.as_numpy_dtype) + probs /= np.linalg.norm(probs, ord=1) + self._quadrature_grid = grid + self._quadrature_probs = probs # Note: by creating the logits as `log(prob)` we ensure that # `self.mixture_distribution.logits` is equivalent to # `math_ops.log(self.mixture_distribution.probs)`. self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(prob), + logits=np.log(probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -357,10 +361,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): validate_args=validate_args, name="interpolated_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip( - interpolate_loc(quadrature_polynomial_degree, + interpolate_loc(len(self._quadrature_grid), self._interpolate_weight, loc), - interpolate_scale(quadrature_polynomial_degree, + interpolate_scale(len(self._quadrature_grid), self._interpolate_weight, scale)))] @@ -416,9 +420,14 @@ class VectorDiffeomixture(distribution_lib.Distribution): return self._interpolated_affine @property - def quadrature_polynomial_degree(self): - """Polynomial largest exponent used for Gauss-Hermite quadrature.""" - return self._degree + def quadrature_grid(self): + """Quadrature grid points.""" + return self._quadrature_grid + + @property + def quadrature_probs(self): + """Quadrature normalized weights.""" + return self._quadrature_probs def _batch_shape_tensor(self): return self._batch_shape_ @@ -454,10 +463,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): seed=distribution_util.gen_new_seed( seed, "vector_diffeomixture")) - # Stride `self._degree` for `batch_size` number of times. + # Stride `quadrature_degree` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * self._degree, - delta=self._degree, + limit=batch_size * len(self.quadrature_probs), + delta=len(self.quadrature_probs), dtype=ids.dtype) weight = array_ops.gather( diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index f8648fe5bf..ebf91e8bb4 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -265,6 +265,9 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access unweighted_loss = losses.sigmoid_cross_entropy( multi_class_labels=processed_labels, logits=logits, reduction=losses.Reduction.NONE) + # Averages loss over classes. + unweighted_loss = math_ops.reduce_mean( + unweighted_loss, axis=-1, keep_dims=True) return head_lib.LossAndLabels( unweighted_loss=unweighted_loss, processed_labels=processed_labels) @@ -294,12 +297,9 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) - # Averages loss over classes. - per_example_loss = math_ops.reduce_mean( - unweighted_loss, axis=-1, keep_dims=True) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( - per_example_loss, weights=weights, reduction=losses.Reduction.SUM) + unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, @@ -309,7 +309,7 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access labels=processed_labels, probabilities=probabilities, weights=weights, - per_example_loss=per_example_loss)) + unweighted_loss=unweighted_loss)) # Train. if train_op_fn is None: @@ -330,16 +330,16 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access loss=training_loss, train_op=train_op_fn(training_loss)) - def _eval_metric_ops(self, labels, probabilities, weights, per_example_loss): + def _eval_metric_ops(self, labels, probabilities, weights, unweighted_loss): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope( - None, 'metrics', [labels, probabilities, weights, per_example_loss]): + None, 'metrics', [labels, probabilities, weights, unweighted_loss]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( - per_example_loss, weights=weights, name=keys.LOSS_MEAN), + unweighted_loss, weights=weights, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc( labels=labels, predictions=probabilities, weights=weights, diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py index dcbe62b497..ec1386af34 100644 --- a/tensorflow/contrib/estimator/python/estimator/head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/head_test.py @@ -80,9 +80,13 @@ def _sigmoid(logits): def _sigmoid_cross_entropy(labels, logits): + """Returns sigmoid cross entropy averaged over classes.""" sigmoid_logits = _sigmoid(logits) - return (-labels * np.log(sigmoid_logits) - -(1 - labels) * np.log(1 - sigmoid_logits)) + unreduced_result = ( + -labels * np.log(sigmoid_logits) + -(1 - labels) * np.log(1 - sigmoid_logits)) + # Mean over classes + return np.mean(unreduced_result, axis=-1, keepdims=True) class MultiLabelHead(test.TestCase): @@ -226,7 +230,7 @@ class MultiLabelHead(test.TestCase): # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits expected_unweighted_loss = np.array( - [[10., 10.], [15., 0.]], dtype=np.float32) + [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32) actual_unweighted_loss, _ = head.create_loss( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.EVAL, @@ -311,10 +315,8 @@ class MultiLabelHead(test.TestCase): labels = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. - expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes - ) + # Sum over examples. + expected_loss = np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) keys = metric_keys.MetricKeys expected_metrics = { # Average loss over examples. @@ -343,10 +345,9 @@ class MultiLabelHead(test.TestCase): labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) / - n_classes + np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) ) keys = metric_keys.MetricKeys expected_metrics = { @@ -377,10 +378,9 @@ class MultiLabelHead(test.TestCase): labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) / - n_classes + np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) ) keys = metric_keys.MetricKeys expected_metrics = { @@ -407,9 +407,9 @@ class MultiLabelHead(test.TestCase): labels = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes + np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) ) keys = metric_keys.MetricKeys @@ -506,7 +506,7 @@ class MultiLabelHead(test.TestCase): # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits expected_unweighted_loss = np.array( - [[10., 10.], [15., 0.]], dtype=np.float32) + [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32) actual_unweighted_loss, _ = head.create_loss( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.TRAIN, diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py index 6074694f8b..4af87b8b47 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py @@ -16,6 +16,11 @@ These methods come from https://arxiv.org/abs/1606.03498 and https://arxiv.org/abs/1706.08500. + +NOTE: This implementation uses the same weights as in +https://github.com/openai/improved-gan/blob/master/inception_score/model.py, +but is more numerically stable and is an unbiased estimator of the true +Inception score even when splitting the inputs into batches. """ from __future__ import absolute_import @@ -54,17 +59,16 @@ __all__ = [ 'classifier_score', 'frechet_inception_distance', 'frechet_classifier_distance', + 'INCEPTION_DEFAULT_IMAGE_SIZE', ] -INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v3_2017_09_13.tar.gz' -INCEPTION_FROZEN_GRAPH = 'frozen_inception_v3.pb' -INCEPTION_V3_INPUT = 'input' -INCEPTION_V3_OUTPUT = 'InceptionV3/Logits/SpatialSqueeze:0' -INCEPTION_V3_FINAL_POOL = 'InceptionV3/Logits/AvgPool_1a_8x8/AvgPool:0' -_INCEPTION_V3_NUM_CLASSES = 1001 -_INCEPTION_V3_FINAL_POOL_SIZE = 2048 -INCEPTION_V3_DEFAULT_IMG_SIZE = 299 +INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v1_2015_12_05.tar.gz' +INCEPTION_FROZEN_GRAPH = 'inceptionv1_for_inception_score.pb' +INCEPTION_INPUT = 'Mul:0' +INCEPTION_OUTPUT = 'logits:0' +INCEPTION_FINAL_POOL = 'pool_3:0' +INCEPTION_DEFAULT_IMAGE_SIZE = 299 def _validate_images(images, image_size): @@ -106,42 +110,33 @@ def _symmetric_matrix_square_root(mat, eps=1e-10): # NOTE: Floating-point inputs are expected to be in [0, 1]. # Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py. def preprocess_image( - image, height=INCEPTION_V3_DEFAULT_IMG_SIZE, - width=INCEPTION_V3_DEFAULT_IMG_SIZE, central_fraction=0.875, scope=None): - """Prepare one image for evaluation. - - If height and width are specified it would output an image with that size by - applying resize_bilinear. + images, height=INCEPTION_DEFAULT_IMAGE_SIZE, + width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None): + """Prepare a batch of images for evaluation. - If central_fraction is specified it would crop the central fraction of the - input image. + This is the preprocessing portion of the graph from + http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz. Args: - image: 3-D Tensor of image. If dtype is tf.float32 then the range should be - [0, 1], otherwise it would converted to tf.float32 assuming that the range - is [0, MAX], where MAX is largest positive representable number for - int(8/16/32) data type (see `tf.image.convert_image_dtype` for details). - height: integer - width: integer - central_fraction: Optional Float, fraction of the image to crop. + images: 3-D or 4-D Tensor of images. Values are in [0, 255]. + height: Integer. Height of resized output image. + width: Integer. Width of resized output image. scope: Optional scope for name_scope. + Returns: - 3-D float Tensor of prepared image. + 3-D or 4-D float Tensor of prepared image(s). Values are in [-1, 1]. """ - with ops.name_scope(scope, 'eval_image', [image, height, width]): - if image.dtype != dtypes.float32: - image = image_ops.convert_image_dtype(image, dtype=dtypes.float32) - # Crop the central region of the image with an area containing 87.5% of - # the original image. - image = image_ops.central_crop(image, central_fraction=central_fraction) - - # Resize the image to the specified height and width. - image = array_ops.expand_dims(image, 0) - image = image_ops.resize_bilinear(image, [height, width], - align_corners=False) - image = array_ops.squeeze(image, [0]) - image = (image - 0.5) * 2.0 - return image + is_single = images.shape.ndims == 3 + with ops.name_scope(scope, 'preprocess', [images, height, width]): + if not images.dtype.is_floating: + images = math_ops.to_float(images) + images = (images - 128.0) / 128.0 + if is_single: + images = array_ops.expand_dims(images, axis=0) + resized = image_ops.resize_bilinear(images, [height, width]) + if is_single: + resized = array_ops.squeeze(resized, axis=0) + return resized def _kl_divergence(p, p_logits, q): @@ -211,9 +206,9 @@ def _default_graph_def_fn(): def run_inception(images, graph_def=None, default_graph_def_fn=_default_graph_def_fn, - image_size=INCEPTION_V3_DEFAULT_IMG_SIZE, - input_tensor=INCEPTION_V3_INPUT, - output_tensor=INCEPTION_V3_OUTPUT): + image_size=INCEPTION_DEFAULT_IMAGE_SIZE, + input_tensor=INCEPTION_INPUT, + output_tensor=INCEPTION_OUTPUT): """Run images through a pretrained Inception classifier. Args: @@ -338,7 +333,7 @@ def classifier_score(images, classifier_fn, num_batches=1): inception_score = functools.partial( classifier_score, classifier_fn=functools.partial( - run_inception, output_tensor=INCEPTION_V3_OUTPUT)) + run_inception, output_tensor=INCEPTION_OUTPUT)) def trace_sqrt_product(sigma, sigma_v): @@ -479,4 +474,4 @@ def frechet_classifier_distance(real_images, frechet_inception_distance = functools.partial( frechet_classifier_distance, classifier_fn=functools.partial( - run_inception, output_tensor=INCEPTION_V3_FINAL_POOL)) + run_inception, output_tensor=INCEPTION_FINAL_POOL)) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py index 30285964a5..81fa2fc0f1 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py @@ -68,7 +68,7 @@ def _expected_trace_sqrt_product(sigma, sigma_v): # A dummy GraphDef string with the minimum number of Ops. graphdef_string = """ node { - name: "input" + name: "Mul" op: "Placeholder" attr { key: "dtype" @@ -97,7 +97,7 @@ node { } } node { - name: "InceptionV3/Logits/SpatialSqueeze" + name: "logits" op: "Placeholder" attr { key: "dtype" @@ -120,7 +120,7 @@ node { } } node { - name: "InceptionV3/Logits/AvgPool_1a_8x8/AvgPool" + name: "pool_3" op: "Placeholder" attr { key: "dtype" @@ -182,7 +182,7 @@ class ClassifierMetricsTest(test.TestCase): img = array_ops.ones([batch_size, 299, 299, 3]) pool = _run_with_mock( classifier_metrics.run_inception, img, - output_tensor=classifier_metrics.INCEPTION_V3_FINAL_POOL) + output_tensor=classifier_metrics.INCEPTION_FINAL_POOL) self.assertTrue(isinstance(pool, ops.Tensor)) pool.shape.assert_is_compatible_with([batch_size, 2048]) @@ -306,7 +306,7 @@ class ClassifierMetricsTest(test.TestCase): """Test `preprocess_image` graph construction.""" incorrectly_sized_image = array_ops.zeros([520, 240, 3]) correct_image = classifier_metrics.preprocess_image( - image=incorrectly_sized_image) + images=incorrectly_sized_image) _run_with_mock(classifier_metrics.run_inception, array_ops.expand_dims(correct_image, 0)) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 633104ace0..13c69d261c 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -30,6 +30,43 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test +class LayerParametersDictTest(test.TestCase): + + def testSetItem(self): + """Ensure insertion, contains, retrieval works for supported key types.""" + with ops.Graph().as_default(): + lp_dict = layer_collection.LayerParametersDict() + + x = array_ops.constant(0) + y0 = array_ops.constant(0) + y1 = array_ops.constant(0) + z0 = array_ops.constant(0) + z1 = array_ops.constant(0) + keys = [x, (y0, y1), [z0, z1]] + for key in keys: + lp_dict[key] = key + + for key in keys: + self.assertTrue(key in lp_dict) + self.assertEqual(lp_dict[key], key) + + def testSetItemOverlap(self): + """Ensure insertion fails if key overlaps with existing key.""" + with ops.Graph().as_default(): + lp_dict = layer_collection.LayerParametersDict() + + x = array_ops.constant(0) + y = array_ops.constant(0) + lp_dict[x] = 'value' + + with self.assertRaises(ValueError): + lp_dict[(x, y)] = 'value' + + # Ensure 'y' wasn't inserted. + self.assertTrue(x in lp_dict) + self.assertFalse(y in lp_dict) + + class LayerCollectionTest(test.TestCase): def testLayerCollectionInit(self): @@ -157,6 +194,36 @@ class LayerCollectionTest(test.TestCase): double_loss = sess.run(lc2.total_sampled_loss()) self.assertAlmostEqual(2 * single_loss, double_loss) + def testLossFunctionByName(self): + """Ensure loss functions can be identified by name.""" + with ops.Graph().as_default(): + logits = linalg_ops.eye(2) + lc = layer_collection.LayerCollection() + + # Create a new loss function by name. + lc.register_categorical_predictive_distribution(logits, name='loss1') + self.assertEqual(1, len(lc.losses)) + + # Add logits to same loss function. + with self.assertRaises(NotImplementedError): + lc.register_categorical_predictive_distribution(logits, name='loss1') + self.assertEqual(1, len(lc.losses)) + + # Add another new loss function. + lc.register_categorical_predictive_distribution(logits, name='loss2') + self.assertEqual(2, len(lc.losses)) + + def testLossFunctionWithoutName(self): + """Ensure loss functions get unique names if 'name' not specified.""" + with ops.Graph().as_default(): + logits = linalg_ops.eye(2) + lc = layer_collection.LayerCollection() + + # Create a new loss function by name. + lc.register_categorical_predictive_distribution(logits) + lc.register_categorical_predictive_distribution(logits) + self.assertEqual(2, len(lc.losses)) + def testRegisterCategoricalPredictiveDistributionBatchSize1(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py index 5f28f57f6a..9325aa1b73 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import numpy as np from tensorflow.contrib.kfac.python.ops import layer_collection as lc -from tensorflow.contrib.kfac.python.ops import loss_functions as lf from tensorflow.contrib.kfac.python.ops import optimizer from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -124,9 +123,8 @@ class OptimizerTest(test.TestCase): def testUpdateVelocities(self): with ops.Graph().as_default(), self.test_session() as sess: layers = lc.LayerCollection() - layers.losses = [ - lf.CategoricalLogitsNegativeLogProbLoss(array_ops.constant([1.0])) - ] + layers.register_categorical_predictive_distribution( + array_ops.constant([1.0])) opt = optimizer.KfacOptimizer( 0.1, 0.2, 0.3, layers, momentum=0.5, momentum_type='regular') x = variable_scope.get_variable('x', initializer=array_ops.ones((2, 2))) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 1b77f5d3ba..0cb55894ad 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -55,6 +55,7 @@ class LayerParametersDict(OrderedDict): super(LayerParametersDict, self).__init__(*args, **kwargs) def __setitem__(self, key, value): + key = self._canonicalize_key(key) tensors = key if isinstance(key, (tuple, list)) else (key,) key_collisions = self._tensors.intersection(tensors) if key_collisions: @@ -63,9 +64,23 @@ class LayerParametersDict(OrderedDict): super(LayerParametersDict, self).__setitem__(key, value) def __delitem__(self, key): + key = self._canonicalize_key(key) self._tensors.remove(key) super(LayerParametersDict, self).__delitem__(key) + def __getitem__(self, key): + key = self._canonicalize_key(key) + return super(LayerParametersDict, self).__getitem__(key) + + def __contains__(self, key): + key = self._canonicalize_key(key) + return super(LayerParametersDict, self).__contains__(key) + + def _canonicalize_key(self, key): + if isinstance(key, (list, tuple)): + return tuple(key) + return key + # TODO(duckworthd): add capability for LayerCollection to be "finalized" # and do this when it gets used by FisherEstimator / KfacOptimizer @@ -94,13 +109,16 @@ class LayerCollection(object): self.fisher_factors = OrderedDict() self._generic_registrations = set() self._graph = graph or ops.get_default_graph() - self.losses = [] + self._loss_dict = {} # {str: LossFunction} self._subgraph = None with variable_scope.variable_scope(None, default_name=name) as scope: self._var_scope = scope.name - reset_internals = __init__ + @property + def losses(self): + """LossFunctions registered with this LayerCollection.""" + return list(self._loss_dict.values()) def register_block(self, layer_key, fisher_block): """Validates and registers the layer_key associated with the fisher_block. @@ -277,7 +295,8 @@ class LayerCollection(object): def register_categorical_predictive_distribution(self, logits, seed=None, - targets=None): + targets=None, + name=None): """Registers a categorical predictive distribution. Args: @@ -288,16 +307,24 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_categorical_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.CategoricalLogitsNegativeLogProbLoss( logits, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def register_normal_predictive_distribution(self, mean, var=0.5, seed=None, - targets=None): + targets=None, + name=None): """Registers a normal predictive distribution. Args: @@ -312,15 +339,23 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_normal_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.NormalMeanNegativeLogProbLoss( mean, var, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def register_multi_bernoulli_predictive_distribution(self, logits, seed=None, - targets=None): + targets=None, + name=None): """Registers a multi-Bernoulli predictive distribution. Args: @@ -331,10 +366,17 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_multi_bernoulli_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.MultiBernoulliNegativeLogProbLoss( logits, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def make_or_get_factor(self, cls, args): with variable_scope.variable_scope(self._var_scope): diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py index 8bb1c83a45..788d2d0b1a 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py @@ -981,9 +981,8 @@ class BaseEstimator( global_step = training_util.create_global_step(g) features, labels = input_fn() self._check_inputs(features, labels) - global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access - with ops.control_dependencies([global_step_read_tensor]): - model_fn_ops = self._get_train_ops(features, labels) + training_util._get_or_create_global_step_read() # pylint: disable=protected-access + model_fn_ops = self._get_train_ops(features, labels) ops.add_to_collection(ops.GraphKeys.LOSSES, model_fn_ops.loss) all_hooks.extend(hooks) all_hooks.extend([ diff --git a/tensorflow/contrib/predictor/predictor.py b/tensorflow/contrib/predictor/predictor.py index dbc0028259..28fa815684 100644 --- a/tensorflow/contrib/predictor/predictor.py +++ b/tensorflow/contrib/predictor/predictor.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - """Abstract base class for all predictors.""" from __future__ import absolute_import @@ -66,8 +65,9 @@ class Predictor(object): expected_keys = set(self.feed_tensors.keys()) unexpected_keys = input_keys - expected_keys if unexpected_keys: - raise ValueError('Got unexpected keys in input_dict: {}'.format( - unexpected_keys)) + raise ValueError( + 'Got unexpected keys in input_dict: {}\nexpected: {}'.format( + unexpected_keys, expected_keys)) feed_dict = {} for key in self.feed_tensors.keys(): diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD index 80a5debe99..0df5ff50c0 100644 --- a/tensorflow/contrib/training/BUILD +++ b/tensorflow/contrib/training/BUILD @@ -26,6 +26,7 @@ py_library( "python/training/resample.py", "python/training/sampling_ops.py", "python/training/sequence_queueing_state_saver.py", + "python/training/sgdr_learning_rate_decay.py", "python/training/training.py", "python/training/tuner.py", ], diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc index 4d2100c59c..58e1a73be6 100644 --- a/tensorflow/core/kernels/split_op.cc +++ b/tensorflow/core/kernels/split_op.cc @@ -167,11 +167,11 @@ class SplitOpCPU : public SplitOpBase { const auto num_threads = context->device()->tensorflow_cpu_worker_threads()->num_threads; // TODO(jewillco): Tune heuristic further. + const auto input_element_count = input_shape.num_elements(); const bool use_parallelism_between_outputs = (num_split >= 4 && - input_shape.num_elements() >= - std::max(num_threads, num_split) * 4096 && - input_shape.num_elements() < num_split * 180 * 1024); + input_element_count >= std::max(num_threads, num_split) * 4096 && + input_element_count < num_split * 180 * 1024); auto range_output_func = [&indices, context, &output_shape, prefix_dim_size, split_dim_output_size, suffix_dim_size, &sizes, @@ -209,7 +209,7 @@ class SplitOpCPU : public SplitOpBase { // Run in parallel, disabling parallelism in functor. Shard(num_split, context->device()->tensorflow_cpu_worker_threads()->workers, - num_split, kint64max, range_output_func); + num_split, input_element_count / num_split, range_output_func); } else { // Run sequentially, but allow internal parallelism in functor. range_output_func(0, num_split); diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc index e2dd66da1e..3316e5fcc9 100644 --- a/tensorflow/core/kernels/split_v_op.cc +++ b/tensorflow/core/kernels/split_v_op.cc @@ -225,11 +225,11 @@ class SplitVOpCPU : public SplitVOpBase { const auto num_threads = context->device()->tensorflow_cpu_worker_threads()->num_threads; // TODO(jewillco): Tune heuristic further. + const auto input_element_count = input_shape.num_elements(); const bool use_parallelism_between_outputs = (num_split >= 4 && - input_shape.num_elements() >= - std::max(num_threads, num_split) * 4096 && - input_shape.num_elements() < num_split * 180 * 1024); + input_element_count >= std::max(num_threads, num_split) * 4096 && + input_element_count < num_split * 180 * 1024); auto range_output_func = [&indices, context, &input_shape, prefix_dim_size, split_dim, &split_sizes_vec, &split_start_points, @@ -267,7 +267,7 @@ class SplitVOpCPU : public SplitVOpBase { // Run in parallel, disabling parallelism in functor. Shard(num_split, context->device()->tensorflow_cpu_worker_threads()->workers, - num_split, kint64max, range_output_func); + num_split, input_element_count / num_split, range_output_func); } else { // Run sequentially, but allow internal parallelism in functor. range_output_func(0, num_split); diff --git a/tensorflow/core/lib/io/zlib_buffers_test.cc b/tensorflow/core/lib/io/zlib_buffers_test.cc index 66ee68a916..156c712db8 100644 --- a/tensorflow/core/lib/io/zlib_buffers_test.cc +++ b/tensorflow/core/lib/io/zlib_buffers_test.cc @@ -68,25 +68,25 @@ void TestAllCombinations(CompressionOptions input_options, for (auto input_buf_size : InputBufferSizes()) { for (auto output_buf_size : OutputBufferSizes()) { std::unique_ptr file_writer; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); string result; ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); - TF_CHECK_OK(out.Append(StringPiece(data))); - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, input_options); - TF_EXPECT_OK(in.ReadNBytes(data.size(), &result)); + TF_ASSERT_OK(in.ReadNBytes(data.size(), &result)); EXPECT_EQ(result, data); } } @@ -118,24 +118,24 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size, string actual_result; string expected_result; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); for (int i = 0; i < num_writes; i++) { - TF_CHECK_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Append(StringPiece(data))); if (with_flush) { - TF_CHECK_OK(out.Flush()); + TF_ASSERT_OK(out.Flush()); } strings::StrAppend(&expected_result, data); } - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, @@ -143,7 +143,7 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size, for (int i = 0; i < num_writes; i++) { string decompressed_output; - TF_EXPECT_OK(in.ReadNBytes(data.size(), &decompressed_output)); + TF_ASSERT_OK(in.ReadNBytes(data.size(), &decompressed_output)); strings::StrAppend(&actual_result, decompressed_output); } @@ -170,19 +170,19 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) { string data = GenTestString(10); std::unique_ptr file_writer; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); string result; ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); - TF_CHECK_OK(out.Append(StringPiece(data))); - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, @@ -192,5 +192,129 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) { CHECK(read_status.error_message().find("inflate() failed") != string::npos); } +void WriteCompressedFile(Env* env, const string& fname, int input_buf_size, + int output_buf_size, + const CompressionOptions& output_options, + const string& data) { + std::unique_ptr file_writer; + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); + + ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, + output_options); + TF_ASSERT_OK(out.Init()); + + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); +} + +void TestTell(CompressionOptions input_options, + CompressionOptions output_options) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/zlib_buffers_test"; + for (auto file_size : NumCopies()) { + string data = GenTestString(file_size); + for (auto input_buf_size : InputBufferSizes()) { + for (auto output_buf_size : OutputBufferSizes()) { + // Write the compressed file. + WriteCompressedFile(env, fname, input_buf_size, output_buf_size, + output_options, data); + + // Boiler-plate to set up ZlibInputStream. + std::unique_ptr file_reader; + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); + std::unique_ptr input_stream( + new RandomAccessInputStream(file_reader.get())); + ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, + input_options); + + string first_half(data, 0, data.size() / 2); + string bytes_read; + + // Read the first half of the uncompressed file and expect that Tell() + // returns half the uncompressed length of the file. + TF_ASSERT_OK(in.ReadNBytes(first_half.size(), &bytes_read)); + EXPECT_EQ(in.Tell(), first_half.size()); + EXPECT_EQ(bytes_read, first_half); + + // Read the remaining half of the uncompressed file and expect that + // Tell() points past the end of file. + string second_half; + TF_ASSERT_OK( + in.ReadNBytes(data.size() - first_half.size(), &second_half)); + EXPECT_EQ(in.Tell(), data.size()); + bytes_read.append(second_half); + + // Expect that the file is correctly read. + EXPECT_EQ(bytes_read, data); + } + } + } +} + +void TestSkipNBytes(CompressionOptions input_options, + CompressionOptions output_options) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/zlib_buffers_test"; + for (auto file_size : NumCopies()) { + string data = GenTestString(file_size); + for (auto input_buf_size : InputBufferSizes()) { + for (auto output_buf_size : OutputBufferSizes()) { + // Write the compressed file. + WriteCompressedFile(env, fname, input_buf_size, output_buf_size, + output_options, data); + + // Boiler-plate to set up ZlibInputStream. + std::unique_ptr file_reader; + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); + std::unique_ptr input_stream( + new RandomAccessInputStream(file_reader.get())); + ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, + input_options); + + size_t data_half_size = data.size() / 2; + string second_half(data, data_half_size, data.size() - data_half_size); + + // Skip past the first half of the file and expect Tell() returns + // correctly. + TF_ASSERT_OK(in.SkipNBytes(data_half_size)); + EXPECT_EQ(in.Tell(), data_half_size); + + // Expect that second half is read correctly and Tell() returns past + // end of file after reading complete file. + string bytes_read; + TF_ASSERT_OK(in.ReadNBytes(second_half.size(), &bytes_read)); + EXPECT_EQ(bytes_read, second_half); + EXPECT_EQ(in.Tell(), data.size()); + } + } + } +} + +TEST(ZlibInputStream, TellDefaultOptions) { + TestTell(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT()); +} + +TEST(ZlibInputStream, TellRawDeflate) { + TestTell(CompressionOptions::RAW(), CompressionOptions::RAW()); +} + +TEST(ZlibInputStream, TellGzip) { + TestTell(CompressionOptions::GZIP(), CompressionOptions::GZIP()); +} + +TEST(ZlibInputStream, SkipNBytesDefaultOptions) { + TestSkipNBytes(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT()); +} + +TEST(ZlibInputStream, SkipNBytesRawDeflate) { + TestSkipNBytes(CompressionOptions::RAW(), CompressionOptions::RAW()); +} + +TEST(ZlibInputStream, SkipNBytesGzip) { + TestSkipNBytes(CompressionOptions::GZIP(), CompressionOptions::GZIP()); +} + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index 4999d5cc90..984fbc2810 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -32,7 +32,8 @@ ZlibInputStream::ZlibInputStream( z_stream_input_(new Bytef[input_buffer_capacity_]), z_stream_output_(new Bytef[output_buffer_capacity_]), zlib_options_(zlib_options), - z_stream_(new z_stream) { + z_stream_(new z_stream), + bytes_read_(0) { InitZlibBuffer(); } @@ -45,6 +46,7 @@ ZlibInputStream::~ZlibInputStream() { Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); InitZlibBuffer(); + bytes_read_ = 0; return Status::OK(); } @@ -127,6 +129,7 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read, result->append(next_unread_byte_, can_read_bytes); next_unread_byte_ += can_read_bytes; } + bytes_read_ += can_read_bytes; return can_read_bytes; } @@ -170,8 +173,7 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) { return Status::OK(); } -// TODO(srbs): Implement this. -int64 ZlibInputStream::Tell() const { return -1; } +int64 ZlibInputStream::Tell() const { return bytes_read_; } Status ZlibInputStream::Inflate() { int error = inflate(z_stream_.get(), zlib_options_.flush_mode); diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 8faa7dcb8f..9c7e14441c 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -132,6 +132,9 @@ class ZlibInputStream : public InputStreamInterface { // Returns the size of [next_unread_byte_, z_stream_->next_out) size_t NumUnreadBytes() const; + // Number of *uncompressed* bytes that have been read from this stream. + int64 bytes_read_; + TF_DISALLOW_COPY_AND_ASSIGN(ZlibInputStream); }; diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py index 5344526b52..88425ea0d0 100644 --- a/tensorflow/examples/learn/mnist.py +++ b/tensorflow/examples/learn/mnist.py @@ -97,6 +97,8 @@ def conv_model(features, labels, mode): def main(unused_args): + tf.logging.set_verbosity(tf.logging.INFO) + ### Download and load MNIST dataset. mnist = tf.contrib.learn.datasets.DATASETS['mnist']('/tmp/mnist') train_input_fn = tf.estimator.inputs.numpy_input_fn( @@ -115,6 +117,7 @@ def main(unused_args): feature_columns = [ tf.feature_column.numeric_column( X_FEATURE, shape=mnist.train.images.shape[1:])] + classifier = tf.estimator.LinearClassifier( feature_columns=feature_columns, n_classes=N_DIGITS) classifier.train(input_fn=train_input_fn, steps=200) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e4e284dcdf..cbeb0b46cb 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3685,7 +3685,10 @@ py_test( size = "medium", srcs = ["training/monitored_session_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], + tags = [ + "no_windows", + "notsan", # b/67945581 + ], deps = [ ":array_ops", ":client_testlib", diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 9e9a7f4c59..ef04f933c5 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -391,14 +391,14 @@ py_test( ], ) -py_test( +cuda_py_test( name = "ops_test", srcs = ["ops_test.py"], - srcs_version = "PY2AND3", - deps = [ + additional_deps = [ ":context", ":execute", ":test", + "//third_party/py/numpy", "//tensorflow/python:array_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", @@ -410,7 +410,6 @@ py_test( "//tensorflow/python:random_ops", "//tensorflow/python:sparse_ops", "//tensorflow/python:tensor_shape", - "//third_party/py/numpy", ], ) diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 6d1a5fe264..f737bfbc15 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -49,7 +49,7 @@ class OpsTest(test_util.TensorFlowTestCase): three = constant_op.constant([[3.]]).as_gpu_tensor() five = constant_op.constant([[5.]]).as_gpu_tensor() product = math_ops.matmul(three, five) - self.assertEqual([[15.0]], product) + self.assertEqual([[15.0]], product.numpy()) def testExecuteStringAttr(self): three = constant_op.constant(3.0) @@ -97,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase): self.skipTest('No GPUs found') with context.device('/gpu:0'): r = constant_op.constant(1) + constant_op.constant(2) - self.assertEqual(r, 3) + self.assertAllEqual(r, 3) def testExecuteListOutputLen1(self): split_dim = constant_op.constant(1) @@ -264,7 +264,7 @@ class OpsTest(test_util.TensorFlowTestCase): # The Shape op kernel on GPU places the output in host memory. value = constant_op.constant([1.]).as_gpu_tensor() shape = array_ops.shape(value) - self.assertEquals([1], shape) + self.assertEqual([1], shape.numpy()) def testRandomUniform(self): scalar_shape = constant_op.constant([], dtype=dtypes.int32) diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 00a57f11dc..2a4d77b1a6 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -707,12 +707,11 @@ class Estimator(object): with ops.Graph().as_default() as g, g.device(self._device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) - global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access + training_util._get_or_create_global_step_read() # pylint: disable=protected-access features, labels = self._get_features_and_labels_from_input_fn( input_fn, model_fn_lib.ModeKeys.TRAIN) - with ops.control_dependencies([global_step_read_tensor]): - estimator_spec = self._call_model_fn( - features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) + estimator_spec = self._call_model_fn( + features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) # Check if the user created a loss summary, and add one if they didn't. # We assume here that the summary is called 'loss'. If it is not, we will # make another one with the name 'loss' to ensure it shows up in the right diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD index 27062adb61..b1c81dd58c 100644 --- a/tensorflow/python/feature_column/BUILD +++ b/tensorflow/python/feature_column/BUILD @@ -86,6 +86,7 @@ py_test( "//tensorflow/python:framework_ops", "//tensorflow/python:lookup_ops", "//tensorflow/python:parsing_ops", + "//tensorflow/python:partitioned_variables", "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 81f4f45fcb..190a25d4d7 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -197,12 +197,13 @@ def input_layer(features, trainable: If `True` also add the variable to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). cols_to_vars: If not `None`, must be a dictionary that will be filled with a - mapping from `_FeatureColumn` to associated `Variable` (or list of - `Variable`, or `PartitionedVariable`. For example, after the call, we - might have cols_to_vars = {_EmbeddingColumn( + mapping from `_FeatureColumn` to list of `Variable`s. For example, after + the call, we might have cols_to_vars = + {_EmbeddingColumn( categorical_column=_HashedCategoricalColumn( key='sparse_feature', hash_bucket_size=5, dtype=tf.string), - dimension=10): [, - 'bias': , + [], + 'bias': [], _NumericColumn( key='numeric_feature2', shape=(2,)): - } - Note that it will also contain a string key 'bias'. If a column creates - no variables, its value will be an empty list. + []} + If a column creates no variables, its value will be an empty list. Note + that cols_to_vars will also contain a string key 'bias' that maps to a + list of Variables. Returns: A `Tensor` which represents predictions/logits of a linear model. Its shape @@ -366,8 +367,12 @@ def linear_model(features, predictions = nn_ops.bias_add( predictions_no_bias, bias, name='weighted_sum') if cols_to_vars is not None: - # Add the bias to cols_to_vars as well. - cols_to_vars['bias'] = bias + # Add the bias to cols_to_vars as well, converting the Variable or + # PartitionedVariable to a list of Variable's. + if isinstance(bias, variables.Variable): + cols_to_vars['bias'] = [bias] + else: # Must be a PartitionedVariable. + cols_to_vars['bias'] = list(bias) return predictions diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 112600439b..e57e9a9836 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -41,6 +41,7 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import parsing_ops +from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test @@ -1354,10 +1355,33 @@ class LinearModelTest(test.TestCase): bias = get_linear_model_bias() price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) - self.assertEqual(cols_to_vars['bias'], bias) + self.assertAllEqual(cols_to_vars['bias'], [bias]) self.assertAllEqual(cols_to_vars[price1], [price1_var]) self.assertAllEqual(cols_to_vars[price2], [price2_var]) + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1', shape=2) + price2 = fc.numeric_column('price2', shape=3) + with ops.Graph().as_default(): + features = { + 'price1': [[1., 2.], [6., 7.]], + 'price2': [[3., 4., 5.], [8., 9., 10.]] + } + cols_to_vars = {} + with variable_scope.variable_scope( + 'linear', + partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)): + fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars) + with _initialized_session(): + self.assertEqual([0.], cols_to_vars['bias'][0].eval()) + # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables. + self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval()) + # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and + # a [1, 1] Variable. + self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval()) + def test_dense_collection(self): price = fc.numeric_column('price') with ops.Graph().as_default() as g: @@ -1761,9 +1785,38 @@ class InputLayerTest(test.TestCase): self.assertEqual(0, len(cols_to_vars[price1])) self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized])) self.assertEqual(1, len(cols_to_vars[some_embedding_column])) - for var in cols_to_vars[some_embedding_column]: - self.assertIsInstance(var, variables_lib.Variable) - self.assertAllEqual(var.shape, [5, 10]) + self.assertIsInstance(cols_to_vars[some_embedding_column][0], + variables_lib.Variable) + self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10]) + + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1') + dense_feature = fc.numeric_column('dense_feature') + dense_feature_bucketized = fc.bucketized_column( + dense_feature, boundaries=[0.]) + some_sparse_column = fc.categorical_column_with_hash_bucket( + 'sparse_feature', hash_bucket_size=5) + some_embedding_column = fc.embedding_column( + some_sparse_column, dimension=10) + with ops.Graph().as_default(): + features = { + 'price1': [[3.], [4.]], + 'dense_feature': [[-1.], [4.]], + 'sparse_feature': [['a'], ['x']], + } + cols_to_vars = {} + all_cols = [price1, dense_feature_bucketized, some_embedding_column] + with variable_scope.variable_scope( + 'input_from_feature_columns', + partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0)): + fc.input_layer(features, all_cols, cols_to_vars=cols_to_vars) + self.assertItemsEqual(list(cols_to_vars.keys()), all_cols) + self.assertEqual(0, len(cols_to_vars[price1])) + self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized])) + self.assertEqual(3, len(cols_to_vars[some_embedding_column])) + self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [2, 10]) + self.assertAllEqual(cols_to_vars[some_embedding_column][1].shape, [2, 10]) + self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10]) def test_column_order(self): price_a = fc.numeric_column('price_a') diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 3ac8a0cb6a..85b875aa3a 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -33,6 +33,7 @@ from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import function_pb2 from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 +from tensorflow.core.framework import op_def_pb2 from tensorflow.core.framework import versions_pb2 from tensorflow.python import pywrap_tensorflow as c_api from tensorflow.python.eager import context @@ -1985,7 +1986,19 @@ class Operation(object): protocol buffer. """ # pylint: enable=line-too-long - return self._op_def + if self._c_op: + with errors.raise_exception_on_not_ok_status() as status: + with c_api_util.tf_buffer() as buf: + # pylint: disable=protected-access + c_api.TF_GraphGetOpDef(self._graph._c_graph, + compat.as_bytes(self.type), buf, status) + # pylint: enable=protected-access + data = c_api.TF_GetBuffer(buf) + op_def = op_def_pb2.OpDef() + op_def.ParseFromString(compat.as_bytes(data)) + return op_def + else: + return self._op_def @property def traceback(self): @@ -2504,7 +2517,14 @@ class Graph(object): # A map from tensor handle to its delete op. self._handle_deleters = {} # Resource container. - self._container = "" + if context.in_graph_mode(): + self._container_prefix = "" + else: + # In Eager mode, isolate resources (particularly ResourceVariables) in + # Graphs by default. This prevents unintended variable sharing. Graph mode + # gets this kind of isolation from Sessions. + self._container_prefix = "eager-execution-%d/" % (uid(),) + self._container = self._container_prefix self._registered_ops = op_def_registry.get_registered_ops() # TODO(skyewm): fold as much of the above as possible into the C @@ -3816,7 +3836,7 @@ class Graph(object): """ original_container = self._container try: - self._container = container_name + self._container = self._container_prefix + container_name yield self._container finally: self._container = original_container diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index f20c808cde..59c0288457 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -504,6 +504,21 @@ class OperationTest(test_util.TensorFlowTestCase): r"num of inputs: 0\) does not have input 1"): x.op._update_input(1, x) # pylint: disable=protected-access + def testOpDef(self): + x = constant_op.constant(0) + y = constant_op.constant(1) + z = x + y + + # Pure Python mode doesn't create OpDefs for constants + if ops._USE_C_API: + self.assertEqual(x.op.op_def.name, "Const") + self.assertEqual(len(x.op.op_def.input_arg), 0) + self.assertEqual(len(x.op.op_def.output_arg), 1) + + self.assertEqual(z.op.op_def.name, "Add") + self.assertEqual(len(z.op.op_def.input_arg), 2) + self.assertEqual(len(z.op.op_def.output_arg), 1) + @test_util.with_c_api class CreateOpTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py index 1b84af6823..c4e16ff628 100644 --- a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py +++ b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py @@ -654,7 +654,8 @@ class RandomShuffleQueueTest(test.TestCase): def testBlockingDequeueFromClosedQueue(self): with self.test_session() as sess: - q = data_flow_ops.RandomShuffleQueue(10, 2, dtypes_lib.float32) + min_size = 2 + q = data_flow_ops.RandomShuffleQueue(10, min_size, dtypes_lib.float32) elems = [10.0, 20.0, 30.0, 40.0] enqueue_op = q.enqueue_many((elems,)) close_op = q.close() @@ -664,20 +665,24 @@ class RandomShuffleQueueTest(test.TestCase): results = [] - def dequeue(): - for _ in elems: - results.append(sess.run(dequeued_t)) + # Manually dequeue until we hit min_size. + results.append(sess.run(dequeued_t)) + results.append(sess.run(dequeued_t)) + + def blocking_dequeue(): + results.append(sess.run(dequeued_t)) + results.append(sess.run(dequeued_t)) + self.assertItemsEqual(elems, results) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): sess.run(dequeued_t) - dequeue_thread = self.checkedThread(target=dequeue) + dequeue_thread = self.checkedThread(target=blocking_dequeue) dequeue_thread.start() - # The close_op should run after the dequeue_thread has blocked. - # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) + # The dequeue thread blocked when it hit the min_size requirement. self.assertEqual(len(results), 2) close_op.run() diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index ec9192b1a0..23676223dc 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -428,6 +428,16 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(1, v1.read_value().numpy()) self.assertEqual(2, v2.read_value().numpy()) + def testDestruction(self): + with context.eager_mode(): + var = resource_variable_ops.ResourceVariable(initial_value=1.0, + name="var8") + var.__del__() + with self.assertRaisesRegexp(errors.NotFoundError, + r"Resource .*\/var8\/.* does not exist."): + resource_variable_ops.destroy_resource_op(var._handle, + ignore_lookup_error=False) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 2c9a3ff19a..dd3f167145 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -427,6 +427,39 @@ class ResourceVariable(variables.Variable): self._constraint = None # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py) + def __del__(self): + if not self._in_graph_mode: + # There is only one ResourceVariable object for each underlying resource + # (cached in the Graph's VariableStore when created with get_variable), so + # it is safe to delete the resource we have a handle to. Each Graph has a + # unique container name in Eager, which prevents resource sharing. + # + # The Graph's VariableStore contains strong references to ResourceVariable + # objects created with get_variable, so this destructor will only be + # callled once the Graph is garbage collected for those objects. However, + # explicitly created ResourceVariables (e.g. through tfe.Variable) may be + # collected earlier. + try: + # We have checked that this ResourceVariable was created in Eager + # mode. However, this destructor may be running in graph mode + # (especially during unit tests). To clean up successfully, we switch + # back into Eager temporarily. + with context.eager_mode(): + with ops.device(self._handle_device): + gen_resource_variable_ops.destroy_resource_op( + self._handle, ignore_lookup_error=True) + except TypeError: + # Suppress some exceptions, mainly for the case when we're running on + # module deletion. Things that can go wrong include the context module + # already being unloaded, self._handle._handle_data no longer being + # valid, and so on. Printing warnings in these cases is silly + # (exceptions raised from __del__ are printed as warnings to stderr). + pass # 'NoneType' object is not callable when the handle has been + # partially unloaded. + except AttributeError: + pass # 'NoneType' object has no attribute 'eager_mode' when context has + # been unloaded. Will catch other module unloads as well. + @property def dtype(self): """The dtype of this variable.""" diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py index defcf33714..176d20bd60 100644 --- a/tensorflow/python/training/adam_test.py +++ b/tensorflow/python/training/adam_test.py @@ -152,53 +152,54 @@ class AdamOptimizerTest(test.TestCase): def doTestBasic(self, use_resource=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - # Initialize variables for numpy implementation. - m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - - if use_resource: - var0 = resource_variable_ops.ResourceVariable( - var0_np, name="var0_%d" % i) - var1 = resource_variable_ops.ResourceVariable( - var1_np, name="var1_%d" % i) - else: - var0 = variables.Variable(var0_np) - var1 = variables.Variable(var1_np) - grads0 = constant_op.constant(grads0_np) - grads1 = constant_op.constant(grads1_np) - - opt = adam.AdamOptimizer() - update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + with self.test_session(graph=ops.Graph()): + # Initialize variables for numpy implementation. + m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) - if context.in_graph_mode(): - self.evaluate(variables.global_variables_initializer()) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + if use_resource: + var0 = resource_variable_ops.ResourceVariable( + var0_np, name="var0_%d" % i) + var1 = resource_variable_ops.ResourceVariable( + var1_np, name="var1_%d" % i) + else: + var0 = variables.Variable(var0_np) + var1 = variables.Variable(var1_np) + grads0 = constant_op.constant(grads0_np) + grads1 = constant_op.constant(grads1_np) - beta1_power, beta2_power = opt._get_beta_accumulators() + opt = adam.AdamOptimizer() + update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - # Run 3 steps of Adam - for t in range(1, 4): if context.in_graph_mode(): - self.evaluate(update) - elif t > 1: - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - self.assertAllCloseAccordingToType(0.9**(t + 1), - self.evaluate(beta1_power)) - self.assertAllCloseAccordingToType(0.999**(t + 1), - self.evaluate(beta2_power)) + beta1_power, beta2_power = opt._get_beta_accumulators() - var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) - var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) + # Run 3 steps of Adam + for t in range(1, 4): + if context.in_graph_mode(): + self.evaluate(update) + elif t > 1: + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + + self.assertAllCloseAccordingToType(0.9**(t + 1), + self.evaluate(beta1_power)) + self.assertAllCloseAccordingToType(0.999**(t + 1), + self.evaluate(beta2_power)) + + var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0) + var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1) - # Validate updated params - self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) - self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) + # Validate updated params + self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) + self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1)) def testBasic(self): with self.test_session(): diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index 07cd67a4b9..aeb8eaffe8 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -479,14 +479,14 @@ class SaverTest(test.TestCase): self.assertEqual(30.0, v2_2.values().eval()) def _SaveAndLoad(self, var_name, var_value, other_value, save_path): - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: var = resource_variable_ops.ResourceVariable(var_value, name=var_name) save = saver_module.Saver({var_name: var}) if context.in_graph_mode(): self.evaluate(var.initializer) val = save.save(sess, save_path) self.assertEqual(save_path, val) - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: var = resource_variable_ops.ResourceVariable(other_value, name=var_name) save = saver_module.Saver({var_name: var}) save.restore(sess, save_path) @@ -619,27 +619,28 @@ class SaverTest(test.TestCase): # Save and reload one Variable named "var0". self._SaveAndLoad("var0", 0.0, 1.0, save_path) for use_tensor in [True, False]: - var = resource_variable_ops.ResourceVariable(1.0, name="var0") - save = saver_module.Saver( - { - var._shared_name: var - }, pad_step_number=pad_step_number) - if context.in_graph_mode(): - self.evaluate(var.initializer) - sess = ops_lib.get_default_session() - else: - sess = None - if use_tensor: - global_step = constant_op.constant(global_step_int) - val = save.save(sess, save_path, global_step=global_step) - else: - val = save.save(sess, save_path, global_step=global_step_int) - if pad_step_number: - expected_save_path = "%s-%s" % (save_path, - "{:08d}".format(global_step_int)) - else: - expected_save_path = "%s-%d" % (save_path, global_step_int) - self.assertEqual(expected_save_path, val) + with self.test_session(graph=ops_lib.Graph()): + var = resource_variable_ops.ResourceVariable(1.0, name="var0") + save = saver_module.Saver( + { + var._shared_name: var + }, pad_step_number=pad_step_number) + if context.in_graph_mode(): + self.evaluate(var.initializer) + sess = ops_lib.get_default_session() + else: + sess = None + if use_tensor: + global_step = constant_op.constant(global_step_int) + val = save.save(sess, save_path, global_step=global_step) + else: + val = save.save(sess, save_path, global_step=global_step_int) + if pad_step_number: + expected_save_path = "%s-%s" % (save_path, + "{:08d}".format(global_step_int)) + else: + expected_save_path = "%s-%d" % (save_path, global_step_int) + self.assertEqual(expected_save_path, val) def testSaveWithGlobalStepWithPadding(self): self.testSaveWithGlobalStep(pad_step_number=True) diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc index 30290c7a16..f2934a79bd 100644 --- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc +++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc @@ -17,12 +17,20 @@ limitations under the License. #include #include +#include +#include +#include +#include +#include +#include #include "tensorflow/core/common_runtime/constant_folding.h" #include "tensorflow/core/common_runtime/shape_refiner.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/graph/subgraph.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/util/command_line_flags.h" @@ -30,33 +38,38 @@ limitations under the License. namespace tensorflow { namespace graph_transforms { +namespace { +using StringPieceSet = std::unordered_set; +template +using StringPieceMap = std::unordered_map; +} // namespace Status ReplaceSendRecvs(const GraphDef& original_graph_def, const GraphDef& rewritten_graph_def, const std::vector& inputs, const std::vector& outputs, GraphDef* output_graph_def) { - std::map original_map; - MapNamesToNodes(original_graph_def, &original_map); - std::map new_node_names; - for (const NodeDef& node : rewritten_graph_def.node()) { - // If the op isn't a Recv, or it was in the original, nothing to do. - if ((node.op() != "_Recv") || (original_map.count(node.name()) == 1)) { - continue; - } - // See if it matches an input from the original. - for (const string& input : inputs) { - // Here we rely on the naming convention for the Recv nodes that - // RewriteGraphForExecution adds in the place of the feed inputs. - string input_prefix = "_recv_" + input + "_"; - if (StringPiece(node.name()).starts_with(input_prefix)) { - // If it does, prepare to rename any inputs that refer to it. - new_node_names[node.name()] = input; - } - } + // recv_node_names serves as a string storage for recv node names. + std::vector recv_node_names(inputs.size()); + StringPieceMap recv_node_map; + StringPieceSet input_nodes; + for (int i = 0; i < inputs.size(); ++i) { + // RewriteGraphForExecution adds a recv node for each input edge. We assume + // here that adding such recv node did not fail. For example, the original + // graph did not already have a node with the name for the new added recv + // node. + TensorId id = ParseTensorName(inputs[i]); + input_nodes.insert(id.first); + string& recv_node_name = recv_node_names[i]; + recv_node_name = strings::StrCat("_recv_", id.first, "_", id.second); + recv_node_map.emplace(recv_node_name, id); + } + + StringPieceMap original_map; + for (const NodeDef& node : original_graph_def.node()) { + original_map.emplace(node.name(), &node); } - std::vector nodes_to_add; for (const NodeDef& node : rewritten_graph_def.node()) { if ((node.op() == "_Send") || (node.op() == "_Recv")) { // If the op is a Send or Recv that wasn't in the original, skip it. @@ -64,55 +77,68 @@ Status ReplaceSendRecvs(const GraphDef& original_graph_def, continue; } } - NodeDef new_node; - new_node = node; - new_node.mutable_input()->Clear(); - for (const string& old_input : node.input()) { - string input_prefix; - string input_node_name; - string input_suffix; - NodeNamePartsFromInput(old_input, &input_prefix, &input_node_name, - &input_suffix); - string new_input; - if (new_node_names.count(input_node_name) > 0) { - new_input = - input_prefix + new_node_names[input_node_name] + input_suffix; - } else { - new_input = old_input; + + NodeDef* new_node = output_graph_def->add_node(); + new_node->MergeFrom(node); + for (int i = 0; i < new_node->input_size(); ++i) { + string& input = *new_node->mutable_input(i); + TensorId id = ParseTensorName(input); + const auto iter = recv_node_map.find(id.first); + if (iter != recv_node_map.end()) { + // The node being substituted is a Recv node, and it has only one + // output. If this input is not a control input, then replace the input + // with the mapped value. Otherwise, replace the node name only. + if (id.second != Graph::kControlSlot) { + CHECK_EQ(id.second, 0); + input = iter->second.ToString(); + } else { + id.first = iter->second.first; + input = id.ToString(); + } } - *(new_node.mutable_input()->Add()) = new_input; } - nodes_to_add.push_back(new_node); - } - for (std::pair entry : new_node_names) { - string removed_node_name = entry.second; - const NodeDef* removed_node = original_map[removed_node_name]; - NodeDef new_node; - new_node = *removed_node; - nodes_to_add.push_back(new_node); + + // RewriteGraphForExecution() did not remove this input node. Remove this + // node name from input_nodes so that a duplicate does not get added to the + // output_graph_def. + auto iter = input_nodes.find(new_node->name()); + if (iter != input_nodes.end()) { + input_nodes.erase(iter); + } } - for (const NodeDef& node : nodes_to_add) { - *output_graph_def->mutable_node()->Add() = node; + // Some input nodes are removed in rewrite_graph_def. Add those nodes to + // output_graph_def. + for (StringPiece name : input_nodes) { + const NodeDef& removed_node = *CHECK_NOTNULL(original_map[name]); + output_graph_def->add_node()->MergeFrom(removed_node); } + return Status::OK(); } Status RemoveUnusedNodes(const GraphDef& input_graph_def, const TransformFuncContext& context, GraphDef* output_graph_def) { - std::map node_map; - MapNamesToNodes(input_graph_def, &node_map); + StringPieceMap node_map; + for (const NodeDef& node : input_graph_def.node()) { + node_map.emplace(node.name(), &node); + } - std::set used_nodes; + std::unordered_set input_names; for (const string& input : context.input_names) { - used_nodes.insert(input); + input_names.insert(ParseTensorName(input)); + } + StringPieceSet used_nodes; + StringPieceSet current_nodes; + for (const string& name : context.output_names) { + TensorId id = ParseTensorName(name); + used_nodes.insert(id.first); + current_nodes.insert(id.first); } - std::vector current_nodes = context.output_names; while (!current_nodes.empty()) { - std::set next_nodes; - for (const string& node_name : current_nodes) { - used_nodes.insert(node_name); + StringPieceSet next_nodes; + for (StringPiece node_name : current_nodes) { if (node_map.count(node_name) == 0) { LOG(ERROR) << "Bad graph structure, no node named '" << node_name << "' found for input lookup"; @@ -120,14 +146,20 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def, node_name, "' found for input lookup"); } const NodeDef& node = *(node_map[node_name]); - for (const string& input_name : node.input()) { - const string& input_node_name = NodeNameFromInput(input_name); - if (used_nodes.count(input_node_name) == 0) { - next_nodes.insert(input_node_name); + for (const string& input : node.input()) { + TensorId id = ParseTensorName(input); + if (input_names.count(id) > 0) { + continue; + } + if (used_nodes.insert(id.first).second) { + next_nodes.insert(id.first); } } } - current_nodes = std::vector(next_nodes.begin(), next_nodes.end()); + current_nodes.swap(next_nodes); + } + for (const TensorId& id : input_names) { + used_nodes.insert(id.first); } FilterGraphDef( input_graph_def, @@ -141,7 +173,7 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def, Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle, shape_inference::InferenceContext* context, PartialTensorShape* shape) { - // The default is already unknown + // The default is already unknown. if (!context->RankKnown(handle)) return Status::OK(); std::vector dims(context->Rank(handle)); @@ -151,47 +183,6 @@ Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle, return PartialTensorShape::MakePartialShape(dims.data(), dims.size(), shape); } -Status ShapeForNode(const TransformFuncContext& context, - const string& node_name, TensorShape* result, - bool* has_shape_specified) { - *has_shape_specified = false; - - // Check to see if we have been given a default for all placeholders. - if (context.params.count("type")) { - if (context.params.at("shape").size() != 1) { - return errors::InvalidArgument( - "You must pass no more than one default 'shape' to " - "fold_constants"); - } - const string& shape_string = context.params.at("shape")[0]; - TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result)); - *has_shape_specified = true; - } - - // See if there's a particular type specified for this placeholder. - if (context.params.count("name") || context.params.count("type_for_name")) { - if (!context.params.count("name") || - !context.params.count("type_for_name") || - (context.params.at("type_for_name").size() != - context.params.at("name").size())) { - return errors::InvalidArgument( - "You must pass a 'shape_for_name' arg for every 'name', e.g. " - "fold_constants(name=foo, shape_for_name=\"2,2,1\", name=bar, " - "shape_for_name=\"1\""); - } - const int name_count = context.params.at("name").size(); - for (int i = 0; i < name_count; ++i) { - if (context.params.at("name")[i] == node_name) { - const string& shape_string = context.params.at("shape_for_name")[i]; - TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result)); - *has_shape_specified = true; - } - } - } - - return Status::OK(); -} - // Converts any sub-graphs that can be resolved into constant expressions into // single Const ops. Status FoldConstants(const GraphDef& input_graph_def, @@ -215,17 +206,6 @@ Status FoldConstants(const GraphDef& input_graph_def, GraphDef cleaned_graph_def; RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def); - // Set specified shapes. - for (NodeDef& node : *cleaned_graph_def.mutable_node()) { - TensorShape shape; - bool has_shape_specified; - TF_RETURN_IF_ERROR( - ShapeForNode(context, node.name(), &shape, &has_shape_specified)); - if (has_shape_specified) { - SetNodeAttr("shape", shape, &node); - } - } - TF_RETURN_IF_ERROR( ImportGraphDef({}, cleaned_graph_def, &input_graph, &shape_refiner)); } else { diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc index fd4188a6a4..41106de008 100644 --- a/tensorflow/tools/graph_transforms/fold_constants_test.cc +++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc @@ -74,6 +74,9 @@ class ConstantFoldingTest : public ::testing::Test { TestConstantFolding(graph_def, {{"placeholder_expect_remains", placeholder_tensor}}, {}, {"output_expect_remains"}, {}); + TestConstantFolding(graph_def, + {{"placeholder_expect_remains:0", placeholder_tensor}}, + {}, {"output_expect_remains:0"}, {}); } void TestOpExclusionAdd() { @@ -256,10 +259,40 @@ class ConstantFoldingTest : public ::testing::Test { EXPECT_EQ(0, node_map.count("new_send")); } + void TestReplaceSendRecvsPrefixNames() { + using namespace ::tensorflow::ops; // NOLINT(build/namespaces) + + auto o_root = tensorflow::Scope::NewRootScope(); + auto a = Placeholder(o_root.WithOpName("placeholder"), DT_FLOAT); + auto b = Placeholder(o_root.WithOpName("placeholder_1"), DT_FLOAT); + auto add_o = Add(o_root.WithOpName("add"), a, b); + GraphDef o_graph_def; + TF_ASSERT_OK(o_root.ToGraphDef(&o_graph_def)); + + auto n_root = tensorflow::Scope::NewRootScope(); + auto c = _Recv(n_root.WithOpName("_recv_placeholder_0"), DT_FLOAT, "", "", + 0, ""); + auto d = _Recv(n_root.WithOpName("_recv_placeholder_1_0"), DT_FLOAT, "", "", + 0, ""); + auto add_n = Add(n_root.WithOpName("add"), c, d); + GraphDef n_graph_def; + TF_ASSERT_OK(n_root.ToGraphDef(&n_graph_def)); + + GraphDef result_graph_def; + TF_ASSERT_OK(graph_transforms::ReplaceSendRecvs( + o_graph_def, n_graph_def, {"placeholder", "placeholder_1"}, {"add"}, + &result_graph_def)); + + std::map node_map; + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(1, node_map.count("placeholder")); + EXPECT_EQ(1, node_map.count("placeholder_1")); + EXPECT_EQ(1, node_map.count("add")); + } + void TestRemoveUnusedNodes() { using namespace ::tensorflow::ops; // NOLINT(build/namespaces) auto root = tensorflow::Scope::NewRootScope(); - using namespace ::tensorflow::ops; // NOLINT(build/namespaces) const int width = 100; @@ -295,6 +328,48 @@ class ConstantFoldingTest : public ::testing::Test { EXPECT_EQ(1, node_map.count("output")); EXPECT_EQ(0, node_map.count("unused")); } + + void TestRemoveUnusedNodesMultipleOutputs() { + using namespace ::tensorflow::ops; // NOLINT(build/namespaces) + auto root = tensorflow::Scope::NewRootScope(); + + // a b + // \ / + // shape_n + // \ / + // c + auto a = Placeholder(root.WithOpName("a"), DT_FLOAT); + auto b = Placeholder(root.WithOpName("b"), DT_FLOAT); + auto shape_n = ShapeN(root.WithOpName("shape_n"), {Output(a), Output(b)}); + auto c = Add(root.WithOpName("c"), shape_n[0], shape_n[1]); + + GraphDef graph_def; + TF_ASSERT_OK(root.ToGraphDef(&graph_def)); + GraphDef result_graph_def; + TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes( + graph_def, {{shape_n[0].name()}, {"c"}}, &result_graph_def)); + + // Only one output of shape_n node is fed input. Hence the graph search + // should propagate to inputs of shape_n. Nothing to remove here. + std::map node_map; + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(1, node_map.count("a")); + EXPECT_EQ(1, node_map.count("b")); + EXPECT_EQ(1, node_map.count("c")); + + result_graph_def.Clear(); + TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes( + graph_def, {{shape_n[0].name(), shape_n[1].name()}, {"c"}}, + &result_graph_def)); + + // Both outputs of shape_n node are fed inputs. shape_n does not function + // and inputs to shape_n should be removed. + node_map.clear(); + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(0, node_map.count("a")); + EXPECT_EQ(0, node_map.count("b")); + EXPECT_EQ(1, node_map.count("c")); + } }; TEST_F(ConstantFoldingTest, TestSimpleAdd) { TestSimpleAdd(); } @@ -309,7 +384,15 @@ TEST_F(ConstantFoldingTest, TestPreserveOutputShapes) { TEST_F(ConstantFoldingTest, TestReplaceSendRecvs) { TestReplaceSendRecvs(); } +TEST_F(ConstantFoldingTest, TestReplaceSendRecvsPrefixNames) { + TestReplaceSendRecvsPrefixNames(); +} + TEST_F(ConstantFoldingTest, TestRemoveUnusedNodes) { TestRemoveUnusedNodes(); } +TEST_F(ConstantFoldingTest, TestRemoveUnusedNodesMultipleOutputs) { + TestRemoveUnusedNodesMultipleOutputs(); +} + } // namespace graph_transforms } // namespace tensorflow diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 02723f3e79..4f0de8f768 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -35,7 +35,7 @@ REQUIRED_PACKAGES = [ 'enum34 >= 1.1.6', 'numpy >= 1.12.1', 'six >= 1.10.0', - 'protobuf >= 3.3.0', + 'protobuf >= 3.4.0', 'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0', ] -- GitLab From 2b91b812ef50384cd0526ea513f1cf585adb6ef7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 16:03:56 -0700 Subject: [PATCH 129/573] Move random tests to a separate subdirectory. This in preparation for refactoring the tests to use a common library. PiperOrigin-RevId: 172670730 --- tensorflow/BUILD | 1 + tensorflow/python/kernel_tests/BUILD | 112 +-------------- tensorflow/python/kernel_tests/random/BUILD | 135 ++++++++++++++++++ .../{ => random}/multinomial_op_big_test.py | 0 .../{ => random}/multinomial_op_test.py | 0 .../{ => random}/random_crop_test.py | 0 .../{ => random}/random_gamma_test.py | 0 .../{ => random}/random_ops_test.py | 0 .../{ => random}/random_poisson_test.py | 0 .../{ => random}/random_shuffle_queue_test.py | 0 10 files changed, 140 insertions(+), 108 deletions(-) create mode 100644 tensorflow/python/kernel_tests/random/BUILD rename tensorflow/python/kernel_tests/{ => random}/multinomial_op_big_test.py (100%) rename tensorflow/python/kernel_tests/{ => random}/multinomial_op_test.py (100%) rename tensorflow/python/kernel_tests/{ => random}/random_crop_test.py (100%) rename tensorflow/python/kernel_tests/{ => random}/random_gamma_test.py (100%) rename tensorflow/python/kernel_tests/{ => random}/random_ops_test.py (100%) rename tensorflow/python/kernel_tests/{ => random}/random_poisson_test.py (100%) rename tensorflow/python/kernel_tests/{ => random}/random_shuffle_queue_test.py (100%) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index a563e3b383..93646ad650 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -501,6 +501,7 @@ filegroup( "//tensorflow/python/kernel_tests:all_files", "//tensorflow/python/kernel_tests/distributions:all_files", "//tensorflow/python/kernel_tests/linalg:all_files", + "//tensorflow/python/kernel_tests/random:all_files", "//tensorflow/python/ops/distributions:all_files", "//tensorflow/python/ops/linalg:all_files", "//tensorflow/python/profiler:all_files", diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 847c078971..dece290f83 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -622,21 +622,6 @@ tf_py_test( ], ) -tf_py_test( - name = "random_shuffle_queue_test", - size = "small", - srcs = ["random_shuffle_queue_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - ], -) - cuda_py_test( name = "resource_variable_ops_test", size = "small", @@ -1538,43 +1523,6 @@ cuda_py_test( ], ) -cuda_py_test( - name = "multinomial_op_test", - size = "small", - srcs = ["multinomial_op_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - ], -) - -cuda_py_test( - name = "multinomial_op_big_test", - size = "medium", - srcs = ["multinomial_op_big_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - ], - shard_count = 3, -) - cuda_py_test( name = "numerics_test", size = "small", @@ -1659,30 +1607,6 @@ cuda_py_test( tags = ["no_windows"], ) -cuda_py_test( - name = "random_crop_test", - size = "small", - srcs = ["random_crop_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:random_ops", - ], -) - -cuda_py_test( - name = "random_ops_test", - size = "medium", - srcs = ["random_ops_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:random_ops", - ], -) - cuda_py_test( name = "reduce_join_op_test", size = "small", @@ -2359,37 +2283,6 @@ cuda_py_test( shard_count = 4, ) -cuda_py_test( - name = "random_gamma_test", - size = "medium", - srcs = ["random_gamma_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - ], - shard_count = 4, -) - -cuda_py_test( - name = "random_poisson_test", - size = "medium", - srcs = ["random_poisson_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - ], -) - cuda_py_test( name = "rnn_test", size = "medium", @@ -2805,7 +2698,10 @@ tf_py_test( "//tensorflow/python:variables", ], shard_count = 3, - tags = ["no_windows_gpu"], + tags = [ + "no_windows_gpu", + "nozapfhahn", + ], ) tf_py_test( diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD new file mode 100644 index 0000000000..88a4ddf7f2 --- /dev/null +++ b/tensorflow/python/kernel_tests/random/BUILD @@ -0,0 +1,135 @@ +# Tests of TensorFlow kernels written using the Python API. + +package( + default_visibility = ["//tensorflow:internal"], +) + +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "tf_py_test") +load("//tensorflow:tensorflow.bzl", "cuda_py_test") +load("//tensorflow:tensorflow.bzl", "sycl_py_test") + +# CPU only tests should use tf_py_test, GPU tests use cuda_py_test +# Please avoid the py_tests and cuda_py_tests (plural) while we +# fix the shared/overbroad dependencies. + +tf_py_test( + name = "random_shuffle_queue_test", + size = "small", + srcs = ["random_shuffle_queue_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:data_flow_ops", + "//tensorflow/python:errors", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", + ], +) + +cuda_py_test( + name = "multinomial_op_test", + size = "small", + srcs = ["multinomial_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:random_ops", + ], +) + +cuda_py_test( + name = "multinomial_op_big_test", + size = "medium", + srcs = ["multinomial_op_big_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:random_ops", + ], + shard_count = 3, +) + +cuda_py_test( + name = "random_crop_test", + size = "small", + srcs = ["random_crop_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:random_ops", + ], +) + +cuda_py_test( + name = "random_ops_test", + size = "medium", + srcs = ["random_ops_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:random_ops", + ], +) + +cuda_py_test( + name = "random_gamma_test", + size = "medium", + srcs = ["random_gamma_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", + "//tensorflow/python:random_ops", + ], + shard_count = 4, + tags = ["nozapfhahn"], +) + +cuda_py_test( + name = "random_poisson_test", + size = "medium", + srcs = ["random_poisson_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", + "//tensorflow/python:random_ops", + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/python/kernel_tests/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py similarity index 100% rename from tensorflow/python/kernel_tests/multinomial_op_big_test.py rename to tensorflow/python/kernel_tests/random/multinomial_op_big_test.py diff --git a/tensorflow/python/kernel_tests/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py similarity index 100% rename from tensorflow/python/kernel_tests/multinomial_op_test.py rename to tensorflow/python/kernel_tests/random/multinomial_op_test.py diff --git a/tensorflow/python/kernel_tests/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_crop_test.py rename to tensorflow/python/kernel_tests/random/random_crop_test.py diff --git a/tensorflow/python/kernel_tests/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_gamma_test.py rename to tensorflow/python/kernel_tests/random/random_gamma_test.py diff --git a/tensorflow/python/kernel_tests/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_ops_test.py rename to tensorflow/python/kernel_tests/random/random_ops_test.py diff --git a/tensorflow/python/kernel_tests/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_poisson_test.py rename to tensorflow/python/kernel_tests/random/random_poisson_test.py diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_shuffle_queue_test.py rename to tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py -- GitLab From bba3957467ad8ba9351b829036120412d5d006cb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 16:25:56 -0700 Subject: [PATCH 130/573] Replace as_gpu_tensor and as_cpu_tensor to gpu and cpu PiperOrigin-RevId: 172673720 --- .../contrib/eager/python/summary_writer.py | 18 +++++++------- tensorflow/contrib/eager/python/tfe_test.py | 4 ++-- tensorflow/python/eager/backprop_test.py | 14 +++++------ tensorflow/python/eager/benchmarks_test.py | 4 ++-- tensorflow/python/eager/core_test.py | 24 +++++++++---------- tensorflow/python/eager/function_test.py | 12 +++++----- tensorflow/python/eager/ops_test.py | 12 +++++----- tensorflow/python/framework/constant_op.py | 2 +- tensorflow/python/framework/ops.py | 11 ++++----- tensorflow/python/ops/array_grad.py | 2 +- 10 files changed, 50 insertions(+), 53 deletions(-) diff --git a/tensorflow/contrib/eager/python/summary_writer.py b/tensorflow/contrib/eager/python/summary_writer.py index 39993558e3..5a698b92c6 100644 --- a/tensorflow/contrib/eager/python/summary_writer.py +++ b/tensorflow/contrib/eager/python/summary_writer.py @@ -32,9 +32,9 @@ from tensorflow.python.ops import summary_op_util from tensorflow.python.ops import variable_scope -def _maybe_as_cpu_tensor(v): +def _maybe_cpu(v): if isinstance(v, (ops.EagerTensor, ops.Tensor)): - return v.as_cpu_tensor() + return v.cpu() else: return v @@ -161,9 +161,9 @@ class SummaryWriter(object): gen_summary_ops.write_summary( self._resource, self._update_global_step_tensor(), - _maybe_as_cpu_tensor(tensor), + _maybe_cpu(tensor), tag, - _maybe_as_cpu_tensor(metadata), + _maybe_cpu(metadata), name=scope) def scalar(self, name, tensor, family=None): @@ -185,7 +185,7 @@ class SummaryWriter(object): name, family, values=[tensor]) as (tag, scope): gen_summary_ops.write_scalar_summary( self._resource, self._update_global_step_tensor(), - tag, _maybe_as_cpu_tensor(tensor), name=scope) + tag, _maybe_cpu(tensor), name=scope) def histogram(self, name, tensor, family=None): """Write a histogram summary. @@ -203,7 +203,7 @@ class SummaryWriter(object): name, family, values=[tensor]) as (tag, scope): gen_summary_ops.write_histogram_summary( self._resource, self._update_global_step_tensor(), - tag, _maybe_as_cpu_tensor(tensor), name=scope) + tag, _maybe_cpu(tensor), name=scope) def image(self, name, tensor, bad_color=None, max_images=3, family=None): """Write an image summary.""" @@ -214,7 +214,7 @@ class SummaryWriter(object): name, family, values=[tensor]) as (tag, scope): gen_summary_ops.write_image_summary( self._resource, self._update_global_step_tensor(), - tag, _maybe_as_cpu_tensor(tensor), bad_color_, max_images, + tag, _maybe_cpu(tensor), bad_color_, max_images, name=scope) def audio(self, name, tensor, sample_rate, max_outputs, family=None): @@ -238,7 +238,7 @@ class SummaryWriter(object): gen_summary_ops.write_audio_summary( self._resource, self._update_global_step_tensor(), tag, - _maybe_as_cpu_tensor(tensor), - sample_rate=_maybe_as_cpu_tensor(sample_rate), + _maybe_cpu(tensor), + sample_rate=_maybe_cpu(sample_rate), max_outputs=max_outputs, name=scope) diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index 3d57a98a2e..eab8958f23 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -75,7 +75,7 @@ class TFETest(test_util.TensorFlowTestCase): self.skipTest('No GPUs available') # tf.Tensor.as_gpu_device() moves a tensor to GPU. - x = constant_op.constant([[1., 2.], [3., 4.]]).as_gpu_tensor() + x = constant_op.constant([[1., 2.], [3., 4.]]).gpu() # Alternatively, tf.device() as a context manager places tensors and # operations. with ops.device('gpu:0'): @@ -85,7 +85,7 @@ class TFETest(test_util.TensorFlowTestCase): reduction_indices = range(x.shape.ndims) m = math_ops.reduce_mean(x, reduction_indices) # m is on GPU, bring it back to CPU and compare. - self.assertEqual(3.5, m.as_cpu_tensor().numpy()) + self.assertEqual(3.5, m.cpu().numpy()) def testListDevices(self): # Expect at least one device. diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 002be95d0f..5161095683 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -209,10 +209,10 @@ class BackpropTest(test.TestCase): def fn(x): with context.device('/gpu:0'): b = constant_op.constant(2.0) - c = math_ops.add(x.as_gpu_tensor(), b) - # TODO(apassos): remove as_cpu_tensor below by making TensorVSPace aware + c = math_ops.add(x.gpu(), b) + # TODO(apassos): remove cpu below by making TensorVSPace aware # of devices. - return math_ops.add(c, constant_op.constant(3.0)).as_cpu_tensor() + return math_ops.add(c, constant_op.constant(3.0)).cpu() grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0] self.assertAllEqual(grad, 1.0) @@ -230,7 +230,7 @@ class BackpropTest(test.TestCase): return v.read_value() self.assertEqual( - backprop.implicit_grad(f)()[0][0].as_cpu_tensor().numpy(), 1.0) + backprop.implicit_grad(f)()[0][0].cpu().numpy(), 1.0) def testCPU(self): @@ -247,7 +247,7 @@ class BackpropTest(test.TestCase): self.skipTest('No GPUs found') def f(a, b): - return a.as_cpu_tensor() + b.as_cpu_tensor() + return a.cpu() + b.cpu() with context.device('/gpu:0'): a = constant_op.constant(1.0) @@ -309,8 +309,8 @@ class BackpropTest(test.TestCase): # back: e (cpu) -> add (cpu) -> c (cpu->gpu) -> add (gpu) -> grad (gpu->cpu) def f(a, b): with context.device('/gpu:0'): - c = math_ops.add(a.as_gpu_tensor(0), b.as_gpu_tensor(0)) - return math_ops.add(c.as_cpu_tensor(), constant_op.constant(3.0)) + c = math_ops.add(a.gpu(0), b.gpu(0)) + return math_ops.add(c.cpu(), constant_op.constant(3.0)) with context.device('/cpu:0'): a = constant_op.constant(1.0) diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index 407d1e979c..ebc9e346c0 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -104,7 +104,7 @@ def benchmark_matmul(shape, n, use_gpu=False): transpose_b = (shape[0] != shape[1]) m = random_ops.random_uniform(shape) if use_gpu: - m = m.as_gpu_tensor() + m = m.gpu() # Warm up the GPU - the very first kernel invocation # seems to require a bunch of setup. math_ops.matmul(m, m, transpose_b=transpose_b) @@ -113,7 +113,7 @@ def benchmark_matmul(shape, n, use_gpu=False): return "MatMul {}: {:30s}".format(shape, s) if not use_gpu: - a = m.as_cpu_tensor().numpy() + a = m.cpu().numpy() b = a.T if transpose_b else a with timer(label("np.dot"), iters=n) as iters: for _ in iters: diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py index 54a0be6dd9..2449162dca 100644 --- a/tensorflow/python/eager/core_test.py +++ b/tensorflow/python/eager/core_test.py @@ -122,13 +122,13 @@ class TFETest(test_util.TensorFlowTestCase): if not context.context().num_gpus(): self.skipTest('No GPUs found') - x = constant_op.constant(1.).as_gpu_tensor() + x = constant_op.constant(1.).gpu() with context.device('gpu:0'): y = constant_op.constant(2.) # Add would fail if t2 were not on GPU result = execute( b'Add', 1, inputs=[x, y], - attrs=('T', x.dtype.as_datatype_enum))[0].as_cpu_tensor().numpy() + attrs=('T', x.dtype.as_datatype_enum))[0].cpu().numpy() self.assertEqual(3, result) def testCopyBetweenDevices(self): @@ -136,26 +136,26 @@ class TFETest(test_util.TensorFlowTestCase): self.skipTest('No GPUs found') x = constant_op.constant([[1., 2.], [3., 4.]]) - x = x.as_cpu_tensor() - x = x.as_gpu_tensor() - x = x.as_gpu_tensor() - x = x.as_cpu_tensor() + x = x.cpu() + x = x.gpu() + x = x.gpu() + x = x.cpu() # Invalid device with self.assertRaises(RuntimeError): - x.as_gpu_tensor(context.context().num_gpus() + 1) + x.gpu(context.context().num_gpus() + 1) def testNumpyForceCPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') cpu = constant_op.constant([[1., 2.], [3., 4.]]) - c2g = cpu.as_gpu_tensor() + c2g = cpu.gpu() self.assertAllEqual(c2g, cpu.numpy()) def testCopyFromCPUToCPU(self): ta = constant_op.constant([[1, 2], [3, 4]]) - tb = ta.as_cpu_tensor() + tb = ta.cpu() self.assertNotEqual(id(ta), id(tb)) self.assertAllEqual(ta, tb.numpy()) @@ -189,8 +189,8 @@ class TFETest(test_util.TensorFlowTestCase): def testMatMulGPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') - three = constant_op.constant([[3.]]).as_gpu_tensor() - five = constant_op.constant([[5.]]).as_gpu_tensor() + three = constant_op.constant([[3.]]).gpu() + five = constant_op.constant([[5.]]).gpu() product = execute( b'MatMul', num_outputs=1, @@ -450,7 +450,7 @@ class TFETest(test_util.TensorFlowTestCase): shape = constant_op.constant([], dtype=dtypes.int32) # x: Run the "TruncatedNormal" op CPU and copy result to GPU. - x = truncated_normal(shape).as_gpu_tensor() + x = truncated_normal(shape).gpu() # y: Explicitly run the "TruncatedNormal" op on GPU. with context.device('gpu:0'): y = truncated_normal(shape) diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index e9e396b49b..fb647f5c21 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -173,9 +173,9 @@ class FunctionTest(test.TestCase): if not context.context().num_gpus(): self.skipTest('No GPUs found') - x = constant_op.constant([1.]).as_gpu_tensor() + x = constant_op.constant([1.]).gpu() f = function.defun(math_ops.add) - y = f(x, x).as_cpu_tensor() + y = f(x, x).cpu() self.assertAllEqual(y, [2.]) def testFunctionHandlesInputsOnDifferentDevices(self): @@ -184,9 +184,9 @@ class FunctionTest(test.TestCase): # The Reshape op requires the shape tensor to be placed in host memory. reshape = function.defun(array_ops.reshape) - value = constant_op.constant([1., 2.]).as_gpu_tensor() + value = constant_op.constant([1., 2.]).gpu() shape = constant_op.constant([2, 1]) - reshaped = reshape(value, shape).as_cpu_tensor() + reshaped = reshape(value, shape).cpu() self.assertAllEqual(reshaped, [[1], [2]]) def testFunctionHandlesInputsPlacedOnTheWrongDeviceGracefully(self): @@ -195,8 +195,8 @@ class FunctionTest(test.TestCase): # The Reshape op requires the shape tensor to be placed in host memory. reshape = function.defun(array_ops.reshape) - value = constant_op.constant([1., 2.]).as_gpu_tensor() - shape = constant_op.constant([2, 1]).as_gpu_tensor() + value = constant_op.constant([1., 2.]).gpu() + shape = constant_op.constant([2, 1]).gpu() with self.assertRaises(errors.InvalidArgumentError): reshape(value, shape) diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index f737bfbc15..2ebb625f9f 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -46,8 +46,8 @@ class OpsTest(test_util.TensorFlowTestCase): def testMatMulGPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') - three = constant_op.constant([[3.]]).as_gpu_tensor() - five = constant_op.constant([[5.]]).as_gpu_tensor() + three = constant_op.constant([[3.]]).gpu() + five = constant_op.constant([[5.]]).gpu() product = math_ops.matmul(three, five) self.assertEqual([[15.0]], product.numpy()) @@ -239,10 +239,10 @@ class OpsTest(test_util.TensorFlowTestCase): # The GPU kernel for the Reshape op requires that the # shape input be on CPU. - value = constant_op.constant([1., 2.]).as_gpu_tensor() + value = constant_op.constant([1., 2.]).gpu() shape = constant_op.constant([2, 1]) reshaped = array_ops.reshape(value, shape) - self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor()) + self.assertAllEqual([[1], [2]], reshaped.cpu()) # And if the shape is in device memory, it should complain # TODO(ashankar): Revisit this - perhaps instead of complaining, @@ -250,7 +250,7 @@ class OpsTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp( errors.InvalidArgumentError, 'cannot compute Reshape as input #1 was expected to be on'): - reshaped = array_ops.reshape(value, shape.as_gpu_tensor()) + reshaped = array_ops.reshape(value, shape.gpu()) def testInvalidInputDataType(self): # Fill requires the first input to be an int32 tensor. @@ -262,7 +262,7 @@ class OpsTest(test_util.TensorFlowTestCase): if not context.context().num_gpus(): self.skipTest('No GPUs found') # The Shape op kernel on GPU places the output in host memory. - value = constant_op.constant([1.]).as_gpu_tensor() + value = constant_op.constant([1.]).gpu() shape = array_ops.shape(value) self.assertEqual([1], shape.numpy()) diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py index 686f5aa6db..34848af53b 100644 --- a/tensorflow/python/framework/constant_op.py +++ b/tensorflow/python/framework/constant_op.py @@ -195,7 +195,7 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False): # We don't have a Fill kernel for bool dtype on GPU. So we first run # Fill on CPU and then copy to GPU if needed. with ops.device("/device:CPU:0"): - x = _eager_fill(shape.as_list(), t.as_cpu_tensor(), ctx) + x = _eager_fill(shape.as_list(), t.cpu(), ctx) return _eager_identity(x, ctx) else: return _eager_fill(shape.as_list(), t, ctx) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index ef0ed8fc53..e7e36941e5 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -607,9 +607,6 @@ class _EagerTensorBase(Tensor): def numpy(self): """Returns a numpy array with the same contents as the Tensor. - The contents of the Tensor must be backed by host memory. The - as_cpu_tensor() method can be used ensure that this is true. - TODO(ashankar,agarwal): Perhaps this should NOT reference the underlying buffer but instead always explicitly copy? Note that currently it may or may not copy based on whether the numpy data is properly aligned or not. @@ -618,7 +615,7 @@ class _EagerTensorBase(Tensor): A numpy array that may share memory with the Tensor object. Any changes to one may be reflected in the other. """ - return self.as_cpu_tensor()._numpy() # pylint: disable=protected-access + return self.cpu()._numpy() # pylint: disable=protected-access def __array__(self): return np.array(self.numpy()) @@ -703,11 +700,11 @@ class _EagerTensorBase(Tensor): """The shape of the tensor as a list.""" return list(self._shape_tuple()) - def as_cpu_tensor(self): + def cpu(self): """A copy of this Tensor with contents backed by host memory.""" return self._copy(context.context(), "CPU:0") - def as_gpu_tensor(self, gpu_index=0): + def gpu(self, gpu_index=0): """A copy of this Tensor with contents backed by memory on the GPU. Arguments: @@ -727,7 +724,7 @@ class _EagerTensorBase(Tensor): if self.dtype != dtypes.bool: raise ValueError( "Non-boolean tensor %s cannot be converted to boolean." % repr(self)) - return bool(self.as_cpu_tensor().numpy()) + return bool(self.cpu().numpy()) def __nonzero__(self): return self.__bool__() diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py index 9f8acb2ae3..2ee298ad44 100644 --- a/tensorflow/python/ops/array_grad.py +++ b/tensorflow/python/ops/array_grad.py @@ -397,7 +397,7 @@ def _GatherV2Grad(op, grad): # For axis 0 gathers, build an appropriately shaped IndexedSlices. if axis_static == 0: if context.in_eager_mode(): - params_tail_shape = params_shape.as_cpu_tensor()[1:] + params_tail_shape = params_shape.cpu()[1:] else: params_tail_shape = params_shape[1:] values_shape = array_ops.concat([indices_size, params_tail_shape], 0) -- GitLab From b1cf67b0f6b9c600a03bbdc3eec4fd7b2b6d2deb Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Wed, 18 Oct 2017 17:26:17 -0700 Subject: [PATCH 131/573] Migrate text_classification.py from .contrib utils to .core. Some usages are left untouched: datasets, VocabularyProcessor. A tracking bug is filed for embed_sequence. Tested by re-running and the loss numbers look similar to the ones before the change. PiperOrigin-RevId: 172681096 --- tensorflow/examples/learn/text_classification.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/learn/text_classification.py b/tensorflow/examples/learn/text_classification.py index 26e6e086b3..ba89c532be 100644 --- a/tensorflow/examples/learn/text_classification.py +++ b/tensorflow/examples/learn/text_classification.py @@ -91,11 +91,11 @@ def rnn_model(features, labels, mode): word_list = tf.unstack(word_vectors, axis=1) # Create a Gated Recurrent Unit cell with hidden size of EMBEDDING_SIZE. - cell = tf.contrib.rnn.GRUCell(EMBEDDING_SIZE) + cell = tf.nn.rnn_cell.GRUCell(EMBEDDING_SIZE) # Create an unrolled Recurrent Neural Networks to length of # MAX_DOCUMENT_LENGTH and passes word_list as inputs for each unit. - _, encoding = tf.contrib.rnn.static_rnn(cell, word_list, dtype=tf.float32) + _, encoding = tf.nn.static_rnn(cell, word_list, dtype=tf.float32) # Given encoding of RNN, take encoding of last step (e.g hidden size of the # neural network of last step) and pass it as features for softmax @@ -107,6 +107,8 @@ def rnn_model(features, labels, mode): def main(unused_argv): global n_words + tf.logging.set_verbosity(tf.logging.INFO) + # Prepare training and testing data dbpedia = tf.contrib.learn.datasets.load_dataset( 'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data) -- GitLab From cdd3b14e47e9d5143569730485ad963936256d45 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 17:50:10 -0700 Subject: [PATCH 132/573] Fix MKL build broken by stray line left behind in transpose_op.h. PiperOrigin-RevId: 172683597 --- tensorflow/core/kernels/transpose_op.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/core/kernels/transpose_op.h b/tensorflow/core/kernels/transpose_op.h index ff9cf5d4ff..ae67592d04 100644 --- a/tensorflow/core/kernels/transpose_op.h +++ b/tensorflow/core/kernels/transpose_op.h @@ -86,7 +86,6 @@ class ConjugateTransposeCpuOp : public TransposeOp { }; #ifdef INTEL_MKL -template class MklConjugateTransposeCpuOp : public TransposeOp { public: explicit MklConjugateTransposeCpuOp(OpKernelConstruction* ctx) -- GitLab From 4e5b3696c6747af8e824fbb47c91e980821d24f3 Mon Sep 17 00:00:00 2001 From: Anna R Date: Wed, 18 Oct 2017 17:53:41 -0700 Subject: [PATCH 133/573] Adding ApiDefMap class to op_gen_lib to read ApiDef proto files. PiperOrigin-RevId: 172683926 --- tensorflow/core/BUILD | 2 + tensorflow/core/framework/op_gen_lib.cc | 202 ++++++++++++- tensorflow/core/framework/op_gen_lib.h | 44 +++ tensorflow/core/framework/op_gen_lib_test.cc | 284 +++++++++++++++++++ 4 files changed, 531 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 94ddd0840d..5ab84fec5b 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -510,6 +510,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":lib", + ":lib_internal", ":op_gen_overrides_proto_cc", ":protos_all_cc", ], @@ -2504,6 +2505,7 @@ tf_cc_test( srcs = ["framework/op_gen_lib_test.cc"], deps = [ ":op_gen_lib", + ":protos_all_cc", ":test", ":test_main", ], diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc index 143da996a1..cfaca897ba 100644 --- a/tensorflow/core/framework/op_gen_lib.cc +++ b/tensorflow/core/framework/op_gen_lib.cc @@ -17,11 +17,12 @@ limitations under the License. #include #include "tensorflow/core/framework/attr_value.pb.h" -#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/framework/op_gen_overrides.pb.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/protobuf.h" namespace tensorflow { @@ -393,4 +394,203 @@ const OpGenOverride* OpGenOverrideMap::ApplyOverride(OpDef* op_def) const { return &proto; } +namespace { + +// Initializes given ApiDef with data in OpDef. +void InitApiDefFromOpDef(const OpDef& op_def, ApiDef* api_def) { + api_def->set_graph_op_name(op_def.name()); + api_def->set_visibility(ApiDef::VISIBLE); + + auto* endpoint = api_def->add_endpoint(); + endpoint->set_name(op_def.name()); + if (op_def.has_deprecation()) { + endpoint->set_deprecation_version(op_def.deprecation().version()); + } + + for (const auto& op_in_arg : op_def.input_arg()) { + auto* api_in_arg = api_def->add_in_arg(); + api_in_arg->set_name(op_in_arg.name()); + api_in_arg->set_rename_to(op_in_arg.name()); + api_in_arg->set_description(op_in_arg.description()); + } + for (const auto& op_out_arg : op_def.output_arg()) { + auto* api_out_arg = api_def->add_out_arg(); + api_out_arg->set_name(op_out_arg.name()); + api_out_arg->set_rename_to(op_out_arg.name()); + api_out_arg->set_description(op_out_arg.description()); + } + for (const auto& op_attr : op_def.attr()) { + auto* api_attr = api_def->add_attr(); + api_attr->set_name(op_attr.name()); + api_attr->set_rename_to(op_attr.name()); + if (op_attr.has_default_value()) { + *api_attr->mutable_default_value() = op_attr.default_value(); + } + api_attr->set_description(op_attr.description()); + } + api_def->set_summary(op_def.summary()); + api_def->set_description(op_def.description()); +} + +// Updates base_arg based on overrides in new_arg. +void MergeArg(ApiDef::Arg* base_arg, const ApiDef::Arg& new_arg) { + if (!new_arg.rename_to().empty()) { + base_arg->set_rename_to(new_arg.rename_to()); + } + if (!new_arg.description().empty()) { + base_arg->set_description(new_arg.description()); + } +} + +// Updates base_attr based on overrides in new_attr. +void MergeAttr(ApiDef::Attr* base_attr, const ApiDef::Attr& new_attr) { + if (!new_attr.rename_to().empty()) { + base_attr->set_rename_to(new_attr.rename_to()); + } + if (new_attr.has_default_value()) { + *base_attr->mutable_default_value() = new_attr.default_value(); + } + if (!new_attr.description().empty()) { + base_attr->set_description(new_attr.description()); + } +} + +// Updates base_api_def based on overrides in new_api_def. +Status MergeApiDefs(ApiDef* base_api_def, const ApiDef& new_api_def) { + // Merge visibility + if (new_api_def.visibility() != ApiDef::DEFAULT_VISIBILITY) { + base_api_def->set_visibility(new_api_def.visibility()); + } + // Merge endpoints + if (new_api_def.endpoint_size() > 0) { + base_api_def->clear_endpoint(); + std::copy( + new_api_def.endpoint().begin(), new_api_def.endpoint().end(), + protobuf::RepeatedFieldBackInserter(base_api_def->mutable_endpoint())); + } + // Merge args + for (const auto& new_arg : new_api_def.in_arg()) { + bool found_base_arg = false; + for (int i = 0; i < base_api_def->in_arg_size(); ++i) { + auto* base_arg = base_api_def->mutable_in_arg(i); + if (base_arg->name() == new_arg.name()) { + MergeArg(base_arg, new_arg); + found_base_arg = true; + break; + } + } + if (!found_base_arg) { + return errors::FailedPrecondition("Argument ", new_arg.name(), + " not defined in base api for ", + base_api_def->graph_op_name()); + } + } + for (const auto& new_arg : new_api_def.out_arg()) { + bool found_base_arg = false; + for (int i = 0; i < base_api_def->out_arg_size(); ++i) { + auto* base_arg = base_api_def->mutable_out_arg(i); + if (base_arg->name() == new_arg.name()) { + MergeArg(base_arg, new_arg); + found_base_arg = true; + break; + } + } + if (!found_base_arg) { + return errors::FailedPrecondition("Argument ", new_arg.name(), + " not defined in base api for ", + base_api_def->graph_op_name()); + } + } + // Merge arg order + if (new_api_def.arg_order_size() > 0) { + base_api_def->clear_arg_order(); + std::copy( + new_api_def.arg_order().begin(), new_api_def.arg_order().end(), + protobuf::RepeatedFieldBackInserter(base_api_def->mutable_arg_order())); + } + // Merge attributes + for (const auto& new_attr : new_api_def.attr()) { + bool found_base_attr = false; + for (int i = 0; i < base_api_def->attr_size(); ++i) { + auto* base_attr = base_api_def->mutable_attr(i); + if (base_attr->name() == new_attr.name()) { + MergeAttr(base_attr, new_attr); + found_base_attr = true; + break; + } + } + if (!found_base_attr) { + return errors::FailedPrecondition("Attribute ", new_attr.name(), + " not defined in base api for ", + base_api_def->graph_op_name()); + } + } + // Merge summary + if (!new_api_def.summary().empty()) { + base_api_def->set_summary(new_api_def.summary()); + } + // Merge description + auto description = new_api_def.description().empty() + ? base_api_def->description() + : new_api_def.description(); + + if (!new_api_def.description_prefix().empty()) { + description = + strings::StrCat(new_api_def.description_prefix(), "\n", description); + } + if (!new_api_def.description_suffix().empty()) { + description = + strings::StrCat(description, "\n", new_api_def.description_suffix()); + } + base_api_def->set_description(description); + return Status::OK(); +} +} // namespace + +ApiDefMap::ApiDefMap(const OpList& op_list) { + for (const auto& op : op_list.op()) { + ApiDef api_def; + InitApiDefFromOpDef(op, &api_def); + map_[op.name()] = api_def; + } +} + +ApiDefMap::~ApiDefMap() {} + +Status ApiDefMap::LoadFileList(Env* env, const std::vector& filenames) { + for (const auto& filename : filenames) { + TF_RETURN_IF_ERROR(LoadFile(env, filename)); + } + return Status::OK(); +} + +Status ApiDefMap::LoadFile(Env* env, const string& filename) { + if (filename.empty()) return Status::OK(); + string contents; + TF_RETURN_IF_ERROR(ReadFileToString(env, filename, &contents)); + TF_RETURN_IF_ERROR(LoadApiDef(contents)); + return Status::OK(); +} + +Status ApiDefMap::LoadApiDef(const string& api_def_file_contents) { + const string contents = PBTxtFromMultiline(api_def_file_contents); + ApiDefs api_defs; + protobuf::TextFormat::ParseFromString(contents, &api_defs); + for (const auto& api_def : api_defs.op()) { + // Check if the op definition is already loaded. + if (map_.find(api_def.graph_op_name()) != map_.end()) { + // Overwrite current api def with data in api_def. + TF_RETURN_IF_ERROR(MergeApiDefs(&map_[api_def.graph_op_name()], api_def)); + } else { + return errors::FailedPrecondition( + "Unexpected ApiDef override: ", api_def.graph_op_name(), + " is not defined in base ApiDef."); + } + } + return Status::OK(); +} + +const tensorflow::ApiDef* ApiDefMap::GetApiDef(const string& name) const { + return gtl::FindOrNull(map_, name); +} } // namespace tensorflow diff --git a/tensorflow/core/framework/op_gen_lib.h b/tensorflow/core/framework/op_gen_lib.h index dbe0a8e190..efb287477b 100644 --- a/tensorflow/core/framework/op_gen_lib.h +++ b/tensorflow/core/framework/op_gen_lib.h @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include "tensorflow/core/framework/api_def.pb.h" +#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/env.h" @@ -74,6 +76,48 @@ class OpGenOverrideMap { std::unordered_map> map_; }; +// Takes a list of files with ApiDefs text protos, and allows you to +// look up the specific ApiDef for any given op. +class ApiDefMap { + public: + // OpList must be a superset of ops of any subsequently loaded + // ApiDef. + explicit ApiDefMap(const OpList& op_list); + ~ApiDefMap(); + + // You can call this method multiple times to load multiple + // sets of files. Api definitions are merged if the same + // op definition is loaded multiple times. Later-loaded + // definitions take precedense. + // ApiDefs loaded from files must contain a subset of ops defined + // in the OpList passed to the constructor. + Status LoadFileList(Env* env, const std::vector& filenames); + + // Load a single file. Api definitions are merged if the same + // op definition is loaded multiple times. Later-loaded + // definitions take precedense. + // ApiDefs loaded from file must contain a subset of ops defined + // in the OpList passed to the constructor. + Status LoadFile(Env* env, const string& filename); + + // Load ApiDefs from string containing ApiDefs text proto. + // api_def_file_contents is expected to be in "multiline format". + // ApiDefs must contain a subset of ops defined in OpsList + // passed to the constructor. + Status LoadApiDef(const string& api_def_file_contents); + + // Look up ApiDef proto based on the given graph op name. + // If graph op name is not in this ApiDefMap, returns nullptr. + // + // Note: Returned ApiDef pointer should stay valid even after calling + // Load* functions defined above. Subsequent calls to Load* might modify + // returned ApiDef contents, but should never remove the ApiDef itself. + const ApiDef* GetApiDef(const string& name) const; + + private: + std::unordered_map map_; +}; + } // namespace tensorflow #endif // TENSORFLOW_FRAMEWORK_OP_GEN_LIB_H_ diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc index cc1d117f38..b7ee6db991 100644 --- a/tensorflow/core/framework/op_gen_lib_test.cc +++ b/tensorflow/core/framework/op_gen_lib_test.cc @@ -15,11 +15,60 @@ limitations under the License. #include "tensorflow/core/framework/op_gen_lib.h" +#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/platform/test.h" namespace tensorflow { namespace { +constexpr char kTestOpList[] = R"(op { + name: "testop" + input_arg { + name: "arg_a" + } + input_arg { + name: "arg_b" + } + output_arg { + name: "arg_c" + } + attr { + name: "attr_a" + } + deprecation { + version: 123 + explanation: "foo" + } +)"; + +constexpr char kTestApiDef[] = R"(op { + graph_op_name: "testop" + visibility: VISIBLE + endpoint { + name: "testop1" + } + in_arg { + name: "arg_a" + } + in_arg { + name: "arg_b" + } + out_arg { + name: "arg_c" + } + attr { + name: "attr_a" + } + summary: "Mock op for testing." + description: <DebugString()); +} + +TEST(OpGenLibTest, ApiDefLoadSingleApiDef) { + const string expected_api_def = R"(op { + graph_op_name: "testop" + visibility: VISIBLE + endpoint { + name: "testop1" + } + in_arg { + name: "arg_a" + rename_to: "arg_a" + } + in_arg { + name: "arg_b" + rename_to: "arg_b" + } + out_arg { + name: "arg_c" + rename_to: "arg_c" + } + attr { + name: "attr_a" + rename_to: "attr_a" + } + summary: "Mock op for testing." + description: "Description for the\ntestop." + arg_order: "arg_a" + arg_order: "arg_b" +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + const auto* api_def = api_map.GetApiDef("testop"); + EXPECT_EQ(1, api_def->endpoint_size()); + EXPECT_EQ("testop1", api_def->endpoint(0).name()); + + ApiDefs api_defs; + *api_defs.add_op() = *api_def; + EXPECT_EQ(expected_api_def, api_defs.DebugString()); +} + +TEST(OpGenLibTest, ApiDefOverrideVisibility) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + endpoint { + name: "testop2" + } +} +)"; + const string api_def2 = R"( +op { + graph_op_name: "testop" + visibility: HIDDEN + endpoint { + name: "testop2" + } +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + auto* api_def = api_map.GetApiDef("testop"); + EXPECT_EQ(ApiDef::VISIBLE, api_def->visibility()); + + // Loading ApiDef with default visibility should + // keep current visibility. + TF_CHECK_OK(api_map.LoadApiDef(api_def1)); + EXPECT_EQ(ApiDef::VISIBLE, api_def->visibility()); + + // Loading ApiDef with non-default visibility, + // should update visibility. + TF_CHECK_OK(api_map.LoadApiDef(api_def2)); + EXPECT_EQ(ApiDef::HIDDEN, api_def->visibility()); +} + +TEST(OpGenLibTest, ApiDefOverrideEndpoints) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + endpoint { + name: "testop2" + } +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + auto* api_def = api_map.GetApiDef("testop"); + ASSERT_EQ(1, api_def->endpoint_size()); + EXPECT_EQ("testop1", api_def->endpoint(0).name()); + + TF_CHECK_OK(api_map.LoadApiDef(api_def1)); + ASSERT_EQ(1, api_def->endpoint_size()); + EXPECT_EQ("testop2", api_def->endpoint(0).name()); +} + +TEST(OpGenLibTest, ApiDefOverrideArgs) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + in_arg { + name: "arg_a" + rename_to: "arg_aa" + } + out_arg { + name: "arg_c" + rename_to: "arg_cc" + } + arg_order: "arg_aa" + arg_order: "arg_b" +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + TF_CHECK_OK(api_map.LoadApiDef(api_def1)); + const auto* api_def = api_map.GetApiDef("testop"); + ASSERT_EQ(2, api_def->in_arg_size()); + EXPECT_EQ("arg_aa", api_def->in_arg(0).rename_to()); + // 2nd in_arg is not renamed + EXPECT_EQ("arg_b", api_def->in_arg(1).rename_to()); + + ASSERT_EQ(1, api_def->out_arg_size()); + EXPECT_EQ("arg_cc", api_def->out_arg(0).rename_to()); + + ASSERT_EQ(2, api_def->arg_order_size()); + EXPECT_EQ("arg_aa", api_def->arg_order(0)); + EXPECT_EQ("arg_b", api_def->arg_order(1)); +} + +TEST(OpGenLibTest, ApiDefOverrideDescriptions) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + summary: "New summary" + description: <summary()); + EXPECT_EQ("A\nNew description\nZ", api_def->description()); + EXPECT_EQ("", api_def->description_prefix()); + EXPECT_EQ("", api_def->description_suffix()); + + TF_CHECK_OK(api_map.LoadApiDef(api_def2)); + EXPECT_EQ("B\nA\nNew description\nZ\nY", api_def->description()); + EXPECT_EQ("", api_def->description_prefix()); + EXPECT_EQ("", api_def->description_suffix()); +} + +TEST(OpGenLibTest, ApiDefInvalidOpInOverride) { + const string api_def1 = R"( +op { + graph_op_name: "different_testop" + endpoint { + name: "testop2" + } +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + auto status = api_map.LoadApiDef(api_def1); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); +} } // namespace } // namespace tensorflow -- GitLab From 274786ab9b825b18dc9a7dd5eb3f312ec6cb92f9 Mon Sep 17 00:00:00 2001 From: Kay Zhu Date: Wed, 18 Oct 2017 17:58:50 -0700 Subject: [PATCH 134/573] [XLA] Run HLO verifier for hlo_evaluator_test and fix one shape mismatch in DoesConcateSimple test case. PiperOrigin-RevId: 172684383 --- tensorflow/compiler/xla/service/BUILD | 2 +- .../xla/service/hlo_evaluator_test.cc | 302 ++++++++---------- 2 files changed, 129 insertions(+), 175 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index fed7bd01f6..1ef329365e 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -117,7 +117,7 @@ tf_cc_test( "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:computation_builder", - "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:hlo_verified_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", # fixdeps: keep "//tensorflow/core:lib", diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc index 5172739624..85477af6fe 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc @@ -30,7 +30,7 @@ limitations under the License. #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/test.h" -#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/util.h" @@ -41,7 +41,7 @@ limitations under the License. namespace xla { namespace { -class HloEvaluatorTest : public HloTestBase { +class HloEvaluatorTest : public HloVerifiedTestBase { protected: HloEvaluatorTest() { evaluator_ = MakeUnique(); } @@ -62,8 +62,7 @@ TEST_F(HloEvaluatorTest, DoesClamp) { auto c3 = b.AddInstruction(HloInstruction::CreateConstant(std::move(value))); auto instruction = b.AddInstruction( HloInstruction::CreateTernary(shape, HloOpcode::kClamp, c1, c2, c3)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); @@ -89,8 +88,7 @@ TEST_F(HloEvaluatorTest, DoesSelect) { b.AddInstruction(HloInstruction::CreateConstant(std::move(on_false))); auto instruction = b.AddInstruction( HloInstruction::CreateTernary(shape, HloOpcode::kSelect, c1, c2, c3)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); @@ -112,8 +110,7 @@ TEST_F(HloEvaluatorTest, DoesAdd) { auto c2 = b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs))); auto instruction = b.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, c1, c2)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); @@ -125,111 +122,100 @@ TEST_F(HloEvaluatorTest, DoesAdd) { // Verifies that HloEvaluator evaluates a HLO instruction that performs // element-wise divide with 2 operands. -TEST_F(HloEvaluatorTest, DoesDivide) { - { - auto lhs_s64 = Literal::CreateR2({{1, 0}, {-100, 4}}); - auto rhs_s64 = Literal::CreateR2({{2, 4}, {4, 4}}); - - Shape shape_s64 = ShapeUtil::MakeShape(S64, {2, 2}); - HloComputation::Builder b(TestName()); - auto c1_s64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_s64))); - auto c2_s64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_s64))); - auto instruction = b.AddInstruction(HloInstruction::CreateBinary( - shape_s64, HloOpcode::kDivide, c1_s64, c2_s64)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - std::unique_ptr result = - evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); - - auto expected = Literal::CreateR2({{0, 0}, {-25, 1}}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } - { - auto lhs_f64 = Literal::CreateR2({{1.0, 0.0}, {-100.0, 4.0}}); - auto rhs_f64 = Literal::CreateR2({{2.2, 4.0}, {4.0, 4.0}}); - - Shape shape_f64 = ShapeUtil::MakeShape(F64, {2, 2}); - HloComputation::Builder b(TestName()); - auto c1_f64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_f64))); - auto c2_f64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_f64))); - auto instruction = b.AddInstruction(HloInstruction::CreateBinary( - shape_f64, HloOpcode::kDivide, c1_f64, c2_f64)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - auto result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); - - auto expected = - Literal::CreateR2({{0.45454545454545453, 0}, {-25, 1}}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } +TEST_F(HloEvaluatorTest, DoesDivideInt64) { + auto lhs_s64 = Literal::CreateR2({{1, 0}, {-100, 4}}); + auto rhs_s64 = Literal::CreateR2({{2, 4}, {4, 4}}); + + Shape shape_s64 = ShapeUtil::MakeShape(S64, {2, 2}); + HloComputation::Builder b(TestName()); + auto c1_s64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_s64))); + auto c2_s64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_s64))); + auto instruction = b.AddInstruction(HloInstruction::CreateBinary( + shape_s64, HloOpcode::kDivide, c1_s64, c2_s64)); + module().AddEntryComputation(b.Build()); + + std::unique_ptr result = + evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); + + auto expected = Literal::CreateR2({{0, 0}, {-25, 1}}); + + LiteralTestUtil::ExpectEqual(*expected, *result); +} +TEST_F(HloEvaluatorTest, DoesDivideDouble) { + auto lhs_f64 = Literal::CreateR2({{1.0, 0.0}, {-100.0, 4.0}}); + auto rhs_f64 = Literal::CreateR2({{2.2, 4.0}, {4.0, 4.0}}); + + Shape shape_f64 = ShapeUtil::MakeShape(F64, {2, 2}); + HloComputation::Builder b(TestName()); + auto c1_f64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_f64))); + auto c2_f64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_f64))); + auto instruction = b.AddInstruction(HloInstruction::CreateBinary( + shape_f64, HloOpcode::kDivide, c1_f64, c2_f64)); + module().AddEntryComputation(b.Build()); + + auto result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); + + auto expected = + Literal::CreateR2({{0.45454545454545453, 0}, {-25, 1}}); + + LiteralTestUtil::ExpectEqual(*expected, *result); } // Verifies that HloEvaluator evaluates a HLO instruction that performs // element-wise abs op with 1 operand. -TEST_F(HloEvaluatorTest, DoesAbs) { - { - auto operand = Literal::CreateR2({{1, -20}, {-100, 4}}); - const Shape& shape = ShapeUtil::MakeShape(S64, {2, 2}); - HloComputation::Builder b(TestName()); - auto c1 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); - auto instruction = b.AddInstruction( - HloInstruction::CreateUnary(shape, HloOpcode::kAbs, c1)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - std::unique_ptr result = - evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); - - auto expected = Literal::CreateR2({{1, 20}, {100, 4}}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } +TEST_F(HloEvaluatorTest, DoesAbsR2) { + auto operand = Literal::CreateR2({{1, -20}, {-100, 4}}); + const Shape& shape = ShapeUtil::MakeShape(S64, {2, 2}); + HloComputation::Builder b(TestName()); + auto c1 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); + auto instruction = + b.AddInstruction(HloInstruction::CreateUnary(shape, HloOpcode::kAbs, c1)); + module().AddEntryComputation(b.Build()); + std::unique_ptr result = + evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); + + auto expected = Literal::CreateR2({{1, 20}, {100, 4}}); + + LiteralTestUtil::ExpectEqual(*expected, *result); +} +TEST_F(HloEvaluatorTest, DoesAbsR0) { // For R0 literal. - { - const Shape& r0 = ShapeUtil::MakeShape(F32, {}); - auto operand = Literal::CreateR0(-1.0f); - HloComputation::Builder b(TestName()); - auto c1 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); - auto instruction = - b.AddInstruction(HloInstruction::CreateUnary(r0, HloOpcode::kAbs, c1)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); - auto expected = Literal::CreateR0(1.0f); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } + const Shape& r0 = ShapeUtil::MakeShape(F32, {}); + auto operand = Literal::CreateR0(-1.0f); + HloComputation::Builder b(TestName()); + auto c1 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); + auto instruction = + b.AddInstruction(HloInstruction::CreateUnary(r0, HloOpcode::kAbs, c1)); + module().AddEntryComputation(b.Build()); + + auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); + auto expected = Literal::CreateR0(1.0f); + LiteralTestUtil::ExpectEqual(*expected, *result); +} +TEST_F(HloEvaluatorTest, DoesAbsR1WithZeroSize) { // For R1 literal with dimension of size 0. - { - Shape empty_r1 = ShapeUtil::MakeShape(F32, {0}); - auto operand = Literal::CreateR1({}); - HloComputation::Builder b(TestName()); - auto c1 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); - auto instruction = b.AddInstruction( - HloInstruction::CreateUnary(empty_r1, HloOpcode::kAbs, c1)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); - auto expected = Literal::CreateR1({}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } -} // namespace + Shape empty_r1 = ShapeUtil::MakeShape(F32, {0}); + auto operand = Literal::CreateR1({}); + HloComputation::Builder b(TestName()); + auto c1 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); + auto instruction = b.AddInstruction( + HloInstruction::CreateUnary(empty_r1, HloOpcode::kAbs, c1)); + module().AddEntryComputation(b.Build()); + + auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); + auto expected = Literal::CreateR1({}); + + LiteralTestUtil::ExpectEqual(*expected, *result); +} // Verifies that HloEvaluator evaluates a HLO Computation with non-parameter nor // constant operands. @@ -253,8 +239,7 @@ TEST_F(HloEvaluatorTest, DoesTraverseInstructions) { b.AddInstruction(HloInstruction::CreateParameter(2, shape, "rhs2")); b.AddInstruction(HloInstruction::CreateBinary(shape, HloOpcode::kAdd, lhs_instruction, param_rhs2)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, args).ConsumeValueOrDie(); @@ -279,8 +264,7 @@ TEST_F(HloEvaluatorTest, DoesReshape) { const int64 permutation[] = {1, 2, 0, 4, 3}; b.AddInstruction( HloInstruction::CreateTranspose(shape, literal_instruction, permutation)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -303,8 +287,7 @@ TEST_F(HloEvaluatorTest, DoesBroadcast) { HloInstruction::CreateConstant(std::move(input_literal))); b.AddInstruction(HloInstruction::CreateBroadcast( output_literal->shape(), literal_instruction, {1, 2})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -324,8 +307,7 @@ TEST_F(HloEvaluatorTest, DoesBroadcastScalar) { b.AddInstruction(HloInstruction::CreateBroadcast( output_literal->shape(), literal_instruction, /*broadcast_dimensions=*/{})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -343,11 +325,10 @@ TEST_F(HloEvaluatorTest, DoesConcatenateSimple) { std::vector operands = {operand1, operand2}; - Shape shape = ShapeUtil::MakeShape(S64, {2, 2}); + Shape shape = ShapeUtil::MakeShape(S64, {4, 2}); b.AddInstruction(HloInstruction::CreateConcatenate(shape, operands, 0)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -370,8 +351,7 @@ TEST_F(HloEvaluatorTest, ConcatenateHandlesShapeWithZeroElement) { Shape shape = ShapeUtil::MakeShape(S64, {2}); b.AddInstruction(HloInstruction::CreateConcatenate(shape, operands, 0)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -392,8 +372,7 @@ TEST_F(HloEvaluatorTest, ConvertWithSameLayout) { HloInstruction* constant = b.AddInstruction( HloInstruction::CreateConstant(std::move(input_literal))); b.AddInstruction(HloInstruction::CreateConvert(expected->shape(), constant)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -414,8 +393,7 @@ TEST_F(HloEvaluatorTest, ConvertWithDifferentLayout) { HloInstruction* constant = b.AddInstruction( HloInstruction::CreateConstant(std::move(input_literal))); b.AddInstruction(HloInstruction::CreateConvert(expected->shape(), constant)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -451,8 +429,7 @@ TEST_F(HloEvaluatorTest, Pad2DIntegerArrayWithZeroDimension) { Shape shape = ShapeUtil::MakeShape(S32, {5, 2}); auto pad_instruction = b.AddInstruction(HloInstruction::CreatePad( shape, operand_instruction, padding_value_instruction, padding_config)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); auto result = evaluator_->Evaluate(pad_instruction).ConsumeValueOrDie(); @@ -479,8 +456,7 @@ TEST_F(HloEvaluatorTest, Pad4DFloatArrayWithInteriorPadding) { CreatePaddingConfig({{{1, 0, 2}}, {{0, 2, 1}}, {{0, 0, 0}}, {{0, 0, 0}}}); b.AddInstruction(HloInstruction::CreatePad( shape, input_instruction, pad_instruction, r4_padding_on_dim0_dim1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -525,8 +501,7 @@ TEST_F(HloEvaluatorTest, NegativePadding2D) { pad_value_instruction, r2_padding_on_dim0_dim1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -572,8 +547,7 @@ TEST_F(HloEvaluatorTest, NegativeAndInteriorPadding2D) { pad_value_instruction, r2_padding_on_dim0_dim1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -609,8 +583,7 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank1) { Shape shape = ShapeUtil::MakeShape(F32, {4, 2}); b.AddInstruction(HloInstruction::CreateBinary( shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -653,8 +626,7 @@ TEST_F(HloEvaluatorTest, DotRank1AndRank2) { Shape shape = ShapeUtil::MakeShape(F32, {2}); b.AddInstruction(HloInstruction::CreateBinary( shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -695,8 +667,7 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank2) { Shape shape = ShapeUtil::MakeShape(F32, {4, 2}); b.AddInstruction(HloInstruction::CreateBinary( shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -749,8 +720,7 @@ TEST_F(HloEvaluatorTest, SimpleConv1D) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 3}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -805,8 +775,7 @@ TEST_F(HloEvaluatorTest, Simple4x4Conv2DWith2x2Kernel) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 4, 4}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -885,8 +854,7 @@ TEST_F(HloEvaluatorTest, Conv2DGeneralDimensions) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 1, 2}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -944,8 +912,7 @@ TEST_F(HloEvaluatorTest, DilatedBaseConv2DWithHighPadding) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 7, 7}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1009,8 +976,7 @@ TEST_F(HloEvaluatorTest, DilatedBaseConv2DWithLowAndHighPadding) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 8, 8}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1082,8 +1048,7 @@ TEST_F(HloEvaluatorTest, const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 9, 3}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1131,15 +1096,14 @@ TEST_F(HloEvaluatorTest, ReduceAdd) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); add_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs)); - HloModule module(TestName()); - auto add_func = module.AddEmbeddedComputation(add_computation.Build()); + auto add_func = module().AddEmbeddedComputation(add_computation.Build()); Shape shape = ShapeUtil::MakeShape(F32, {2}); b.AddInstruction( HloInstruction::CreateReduce(shape, arg_instruction, init_value, /*dimensions_to_reduce=*/{1}, add_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1175,8 +1139,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowMax) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); max_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kMaximum, param_lhs, param_rhs)); - HloModule module(TestName()); - auto max_func = module.AddEmbeddedComputation(max_computation.Build()); + auto max_func = module().AddEmbeddedComputation(max_computation.Build()); Window window; WindowDimension dim; @@ -1193,7 +1156,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowMax) { b.AddInstruction(HloInstruction::CreateReduceWindow( shape, arg_instruction, init_value, window, max_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1227,8 +1190,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); add_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs)); - HloModule module(TestName()); - auto add_func = module.AddEmbeddedComputation(add_computation.Build()); + auto add_func = module().AddEmbeddedComputation(add_computation.Build()); Window window; WindowDimension dim; @@ -1251,7 +1213,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd) { b.AddInstruction(HloInstruction::CreateReduceWindow( shape, arg_instruction, init_value, window, add_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1281,8 +1243,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd6D) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); add_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs)); - HloModule module(TestName()); - auto add_func = module.AddEmbeddedComputation(add_computation.Build()); + auto add_func = module().AddEmbeddedComputation(add_computation.Build()); Window window; @@ -1313,7 +1274,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd6D) { b.AddInstruction(HloInstruction::CreateReduceWindow( shape, arg_instruction, init_value, window, add_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1344,8 +1305,7 @@ TEST_F(HloEvaluatorTest, StridedSlice) { /*start_indices=*/{0, 2}, /*limit_indices=*/{3, 5}, /*strides=*/{2, 3})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1379,8 +1339,7 @@ TEST_F(HloEvaluatorTest, DynamicSlice) { Shape shape = ShapeUtil::MakeShape(F32, {2, 3}); b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand, start_indices, {2, 3})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1416,8 +1375,7 @@ TEST_F(HloEvaluatorTest, DynamicSliceModSlice) { Shape shape = ShapeUtil::MakeShape(F32, {2, 3}); b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand, start_indices, {2, 3})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1454,8 +1412,7 @@ TEST_F(HloEvaluatorTest, DynamicSliceUpdate) { Shape shape = ShapeUtil::MakeShape(F64, {2, 3}); b.AddInstruction(HloInstruction::CreateDynamicUpdateSlice( shape, operand, update, start_indices)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1491,8 +1448,7 @@ TEST_F(HloEvaluatorTest, SetAndGetTuples) { Shape shape = ShapeUtil::MakeShape(F64, {2, 3}); b.AddInstruction(HloInstruction::CreateGetTupleElement(shape, tuple, 1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1531,8 +1487,7 @@ TEST_F(HloEvaluatorTest, SetAndGetNestedTuples) { b.AddInstruction( HloInstruction::CreateGetTupleElement(tuple2->shape(), outer_tuple, 1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1572,8 +1527,7 @@ TEST_F(HloEvaluatorTest, Reverse) { const Shape shape = ShapeUtil::MakeShape(F32, {4, 3, 2, 1}); b.AddInstruction(HloInstruction::CreateReverse(shape, operand, {0, 1})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); -- GitLab From 6c297fa9d5a0add0e38aceaceb57b0c6d83e0aca Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Wed, 18 Oct 2017 17:59:33 -0700 Subject: [PATCH 135/573] Disable constant folding when propagating shapes through functions if requested. PiperOrigin-RevId: 172684434 --- tensorflow/core/common_runtime/shape_refiner.cc | 1 + tensorflow/core/common_runtime/shape_refiner.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc index 2a0bdc9a7b..1ed5eb3f22 100644 --- a/tensorflow/core/common_runtime/shape_refiner.cc +++ b/tensorflow/core/common_runtime/shape_refiner.cc @@ -148,6 +148,7 @@ Status ShapeRefiner::InferShapesForFunction( } ShapeRefiner refiner(graph.versions().producer(), &function_library); + refiner.set_disable_constant_propagation(disable_constant_propagation_); refiner.set_function_library_for_shape_inference(&function_library); if (keep_nested_shapes) refiner.set_keep_nested_shape_inferences(); diff --git a/tensorflow/core/common_runtime/shape_refiner.h b/tensorflow/core/common_runtime/shape_refiner.h index bf4c6d8891..d1288d671e 100644 --- a/tensorflow/core/common_runtime/shape_refiner.h +++ b/tensorflow/core/common_runtime/shape_refiner.h @@ -206,7 +206,7 @@ class ShapeRefiner { // - outer_context will contain output shapes inferred from input shapes // - outer_context will contain nested inferences collection, iff // keep_nested_shapes is true - static Status InferShapesForFunction( + Status InferShapesForFunction( const tensorflow::FunctionLibraryDefinition& function_library, const tensorflow::FunctionDef& function_def, bool keep_nested_shapes, ExtendedInferenceContext* outer_context); -- GitLab From b8f8a3d3660c75a4034bbe69a766d481638a6a4e Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Wed, 18 Oct 2017 20:28:27 -0700 Subject: [PATCH 136/573] Fix the build file and a memory issue for text_classification_character_cnn.py. PiperOrigin-RevId: 172695522 --- tensorflow/BUILD | 2 +- .../learn/text_classification_character_cnn.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 93646ad650..e351037abb 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -123,7 +123,7 @@ config_setting( config_setting( name = "ios_x86_64", values = { - "cc_target_os": "apple", + "crosstool_top": "//tools/osx/crosstool:crosstool", "cpu": "ios_x86_64", }, visibility = ["//visibility:public"], diff --git a/tensorflow/examples/learn/text_classification_character_cnn.py b/tensorflow/examples/learn/text_classification_character_cnn.py index 5f7c8e7371..363ff00362 100644 --- a/tensorflow/examples/learn/text_classification_character_cnn.py +++ b/tensorflow/examples/learn/text_classification_character_cnn.py @@ -30,7 +30,6 @@ import sys import numpy as np import pandas -from sklearn import metrics import tensorflow as tf FLAGS = None @@ -106,6 +105,8 @@ def char_cnn_model(features, labels, mode): def main(unused_argv): + tf.logging.set_verbosity(tf.logging.INFO) + # Prepare training and testing data dbpedia = tf.contrib.learn.datasets.load_dataset( 'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data, size='large') @@ -130,7 +131,7 @@ def main(unused_argv): train_input_fn = tf.estimator.inputs.numpy_input_fn( x={CHARS_FEATURE: x_train}, y=y_train, - batch_size=len(x_train), + batch_size=128, num_epochs=None, shuffle=True) classifier.train(input_fn=train_input_fn, steps=100) @@ -145,13 +146,9 @@ def main(unused_argv): y_predicted = np.array(list(p['class'] for p in predictions)) y_predicted = y_predicted.reshape(np.array(y_test).shape) - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - # Score with tensorflow. scores = classifier.evaluate(input_fn=test_input_fn) - print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) + print('Accuracy: {0:f}'.format(scores['accuracy'])) if __name__ == '__main__': -- GitLab From 76ee352ccdb6927ca961c100b21b31f4431134b0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 18 Oct 2017 20:30:20 -0700 Subject: [PATCH 137/573] Multi-minibatch support for tf.contrib.kfac.fisher_blocks.FullyConnectedDiagonalFB. PiperOrigin-RevId: 172695625 --- .../python/kernel_tests/fisher_blocks_test.py | 146 +++++++++++++++++ .../kernel_tests/layer_collection_test.py | 7 +- .../contrib/kfac/python/ops/fisher_blocks.py | 150 +++++++++++++++--- .../contrib/kfac/python/ops/fisher_factors.py | 19 ++- .../kfac/python/ops/layer_collection.py | 6 +- 5 files changed, 298 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py index f48d1980ba..9b13756e62 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py @@ -209,6 +209,152 @@ class NaiveDiagonalFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +class FullyConnectedDiagonalFB(test.TestCase): + + def setUp(self): + super(FullyConnectedDiagonalFB, self).setUp() + + self.batch_size = 4 + self.input_size = 6 + self.output_size = 3 + + self.inputs = np.random.randn(self.batch_size, self.input_size).astype( + np.float32) + self.outputs = np.zeros([self.batch_size, self.output_size]).astype( + np.float32) + self.output_grads = np.random.randn(self.batch_size, + self.output_size).astype(np.float32) + self.w = np.random.randn(self.input_size, self.output_size).astype( + np.float32) + self.b = np.random.randn(self.output_size).astype(np.float32) + + def fisherApprox(self, has_bias=False): + """Fisher approximation using default inputs.""" + if has_bias: + inputs = np.concatenate( + [self.inputs, np.ones([self.batch_size, 1])], axis=1) + else: + inputs = self.inputs + return self.buildDiagonalFisherApproximation(inputs, self.output_grads) + + def buildDiagonalFisherApproximation(self, inputs, output_grads): + """Builds explicit diagonal Fisher approximation. + + Fisher's diagonal is (d loss / d w)'s elements squared for + d/dw = E[outer(input, output_grad)] + + where the expectation is taken over examples. + + Args: + inputs: np.array of shape [batch_size, input_size]. + output_grads: np.array of shape [batch_size, output_size]. + + Returns: + Diagonal np.array of shape [num_params, num_params] for num_params = + input_size * output_size. + """ + batch_size = inputs.shape[0] + assert output_grads.shape[0] == batch_size + input_size = inputs.shape[1] + output_size = output_grads.shape[1] + fisher_diag = np.zeros((input_size, output_size)) + for i in range(batch_size): + fisher_diag += np.square(np.outer(inputs[i], output_grads[i])) + return np.diag(fisher_diag.flatten()) / batch_size + + def testMultiply(self): + result, _ = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct Fisher-vector product. + expected_result = self.fisherApprox().dot(self.w.flatten()) + expected_result = expected_result.reshape( + [self.input_size, self.output_size]) + + self.assertAllClose(expected_result, result) + + def testMultiplyInverse(self): + _, result = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct inverse Fisher-vector product. + expected_result = np.linalg.inv(self.fisherApprox()).dot(self.w.flatten()) + expected_result = expected_result.reshape( + [self.input_size, self.output_size]) + + self.assertAllClose(expected_result, result) + + def testRegisterAdditionalMinibatch(self): + """Ensure 1 big minibatch and 2 small minibatches are equivalent.""" + multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps( + self.w, [self.inputs], [self.outputs], [self.output_grads]) + multiply_result_small, multiply_inverse_result_small = ( + self.runFisherBlockOps(self.w, + np.split(self.inputs, 2), + np.split(self.outputs, 2), + np.split(self.output_grads, 2))) + + self.assertAllClose(multiply_result_big, multiply_result_small) + self.assertAllClose(multiply_inverse_result_big, + multiply_inverse_result_small) + + def testMultiplyHasBias(self): + result, _ = self.runFisherBlockOps((self.w, self.b), [self.inputs], + [self.outputs], [self.output_grads]) + expected_result = self.fisherApprox(True).dot( + np.concatenate([self.w.flatten(), self.b.flatten()])) + expected_result = expected_result.reshape( + [self.input_size + 1, self.output_size]) + expected_result = (expected_result[:-1], expected_result[-1]) + + self.assertEqual(len(result), 2) + self.assertAllClose(expected_result[0], result[0]) + self.assertAllClose(expected_result[1], result[1]) + + def runFisherBlockOps(self, params, inputs, outputs, output_grads): + """Run Ops guaranteed by FisherBlock interface. + + Args: + params: Tensor or 2-tuple of Tensors. Represents weights or weights and + bias of this layer. + inputs: list of Tensors of shape [batch_size, input_size]. Inputs to + layer. + outputs: list of Tensors of shape [batch_size, output_size]. + Preactivations produced by layer. + output_grads: list of Tensors of shape [batch_size, output_size]. + Gradient of loss with respect to 'outputs'. + + Returns: + multiply_result: Result of FisherBlock.multiply(params) + multiply_inverse_result: Result of FisherBlock.multiply_inverse(params) + """ + + def _as_tensors(tensor_or_tuple): + if isinstance(tensor_or_tuple, (tuple, list)): + return tuple(ops.convert_to_tensor(t) for t in tensor_or_tuple) + return ops.convert_to_tensor(tensor_or_tuple) + + with ops.Graph().as_default(), self.test_session() as sess: + inputs = [_as_tensors(i) for i in inputs] + outputs = [_as_tensors(o) for o in outputs] + output_grads = [_as_tensors(og) for og in output_grads] + params = _as_tensors(params) + + block = fb.FullyConnectedDiagonalFB( + lc.LayerCollection(), has_bias=isinstance(params, (tuple, list))) + for (i, o) in zip(inputs, outputs): + block.register_additional_minibatch(i, o) + + block.instantiate_factors((output_grads,), damping=0.0) + + sess.run(tf_variables.global_variables_initializer()) + sess.run(block._factor.make_covariance_update_op(0.0)) + multiply_result = sess.run(block.multiply(params)) + multiply_inverse_result = sess.run(block.multiply_inverse(params)) + + return multiply_result, multiply_inverse_result + + class FullyConnectedKFACBasicFBTest(test.TestCase): def testFullyConnectedKFACBasicFBInit(self): diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 13c69d261c..53d40da586 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -81,6 +81,11 @@ class LayerCollectionTest(test.TestCase): lc = layer_collection.LayerCollection() lc.register_fully_connected( array_ops.constant(1), array_ops.constant(2), array_ops.constant(3)) + lc.register_fully_connected( + array_ops.constant(1), + array_ops.constant(2), + array_ops.constant(3), + approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_conv2d( array_ops.constant(4), [1, 1, 1, 1], 'SAME', array_ops.ones((1, 1, 1, 1)), array_ops.constant(3)) @@ -91,7 +96,7 @@ class LayerCollectionTest(test.TestCase): 16, approx=layer_collection.APPROX_DIAGONAL_NAME) - self.assertEqual(4, len(lc.get_blocks())) + self.assertEqual(5, len(lc.get_blocks())) def testRegisterBlocksMultipleRegistrations(self): with ops.Graph().as_default(): diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 9d8bb8c8ce..6cca2272d7 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -12,7 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""FisherBlock definitions.""" +"""FisherBlock definitions. + +This library contains classes for estimating blocks in a model's Fisher +Information matrix. Suppose one has a model that parameterizes a posterior +distribution over 'y' given 'x' with parameters 'params', p(y | x, params). Its +Fisher Information matrix is given by, + + F(params) = E[ v(x, y, params) v(x, y, params)^T ] + +where, + + v(x, y, params) = (d / d params) log p(y | x, params) + +and the expectation is taken with respect to the data's distribution for 'x' and +the model's posterior distribution for 'y', + + x ~ p(x) + y ~ p(y | x, params) + +""" from __future__ import absolute_import from __future__ import division @@ -133,8 +152,9 @@ class FullFB(FisherBlock): def multiply(self, vector): vector_flat = utils.tensors_to_column(vector) - out_flat = (math_ops.matmul(self._factor.get_cov(), vector_flat) + - self._damping * vector_flat) + out_flat = ( + math_ops.matmul(self._factor.get_cov(), vector_flat) + + self._damping * vector_flat) return utils.column_to_tensors(vector, out_flat) def full_fisher_block(self): @@ -193,54 +213,105 @@ class NaiveDiagonalFB(FisherBlock): class FullyConnectedDiagonalFB(FisherBlock): """FisherBlock for fully-connected (dense) layers using a diagonal approx. - Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator - that is computed using the well-known trick. - """ + Estimates the Fisher Information matrix's diagonal entries for a fully + connected layer. Unlike NaiveDiagonalFB this uses the low-variance "sum of + squares" estimator. - # TODO(jamesmartens): add units tests for this class + Let 'params' be a vector parameterizing a model and 'i' an arbitrary index + into it. We are interested in Fisher(params)[i, i]. This is, - def __init__(self, layer_collection, inputs, outputs, has_bias=False): + Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] + = E[ v(x, y, params)[i] ^ 2 ] + + Consider fully connected layer in this model with (unshared) weight matrix + 'w'. For an example 'x' that produces layer inputs 'a' and output + preactivations 's', + + v(x, y, w) = vec( x (d loss / d s)^T ) + + This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding + to the layer's parameters 'w'. + """ + + def __init__(self, layer_collection, has_bias=False): """Creates a FullyConnectedDiagonalFB block. Args: layer_collection: The collection of all layers in the K-FAC approximate Fisher information matrix to which this FisherBlock belongs. - inputs: The Tensor of input activations to this layer. - outputs: The Tensor of output pre-activations from this layer. has_bias: Whether the component Kronecker factors have an additive bias. (Default: False) """ - self._inputs = inputs - self._outputs = outputs + self._inputs = [] + self._outputs = [] self._has_bias = has_bias super(FullyConnectedDiagonalFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + self._damping = damping self._factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedDiagonalFactor, (self._inputs, grads_list, - self._has_bias)) + fisher_factors.FullyConnectedDiagonalFactor, + (inputs, grads_list, self._has_bias)) def multiply_inverse(self, vector): + """Approximate damped inverse Fisher-vector product. + + Args: + vector: Tensor or 2-tuple of Tensors. if self._has_bias, Tensor of shape + [input_size, output_size] corresponding to layer's weights. If not, a + 2-tuple of the former and a Tensor of shape [output_size] corresponding + to the layer's bias. + + Returns: + Tensor of the same shape, corresponding to the inverse Fisher-vector + product. + """ reshaped_vect = utils.layer_params_to_mat2d(vector) reshaped_out = reshaped_vect / (self._factor.get_cov() + self._damping) return utils.mat2d_to_layer_params(vector, reshaped_out) def multiply(self, vector): + """Approximate damped Fisher-vector product. + + Args: + vector: Tensor or 2-tuple of Tensors. if self._has_bias, Tensor of shape + [input_size, output_size] corresponding to layer's weights. If not, a + 2-tuple of the former and a Tensor of shape [output_size] corresponding + to the layer's bias. + + Returns: + Tensor of the same shape, corresponding to the Fisher-vector product. + """ reshaped_vect = utils.layer_params_to_mat2d(vector) reshaped_out = reshaped_vect * (self._factor.get_cov() + self._damping) return utils.mat2d_to_layer_params(vector, reshaped_out) def tensors_to_compute_grads(self): + """Tensors to compute derivative of loss with respect to.""" return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, input_size]. Inputs to the + matrix-multiply. + outputs: Tensor of shape [batch_size, output_size]. Layer preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + class ConvDiagonalFB(FisherBlock): """FisherBlock for convolutional layers using a diagonal approx. Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator. """ + # TODO(jamesmartens): add units tests for this class def __init__(self, layer_collection, params, inputs, outputs, strides, @@ -271,14 +342,14 @@ class ConvDiagonalFB(FisherBlock): self._filter_shape = tuple(fltr.shape.as_list()) input_shape = tuple(inputs.shape.as_list()) - self._num_locations = (input_shape[1] * input_shape[2] - // (strides[1] * strides[2])) + self._num_locations = ( + input_shape[1] * input_shape[2] // (strides[1] * strides[2])) super(ConvDiagonalFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): if NORMALIZE_DAMPING_POWER: - damping /= self._num_locations ** NORMALIZE_DAMPING_POWER + damping /= self._num_locations**NORMALIZE_DAMPING_POWER self._damping = damping self._factor = self._layer_collection.make_or_get_factor( @@ -345,10 +416,12 @@ class KroneckerProductFB(FisherBlock): left_factor = self._input_factor.get_cov() right_factor = self._output_factor.get_cov() reshaped_vector = utils.layer_params_to_mat2d(vector) - reshaped_out = (math_ops.matmul(reshaped_vector, right_factor) + - self._output_damping * reshaped_vector) - reshaped_out = (math_ops.matmul(left_factor, reshaped_out) + - self._input_damping * reshaped_out) + reshaped_out = ( + math_ops.matmul(reshaped_vector, right_factor) + + self._output_damping * reshaped_vector) + reshaped_out = ( + math_ops.matmul(left_factor, reshaped_out) + + self._input_damping * reshaped_out) if self._renorm_coeff != 1.0: reshaped_out *= math_ops.cast( self._renorm_coeff, dtype=reshaped_out.dtype) @@ -394,8 +467,8 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB): def instantiate_factors(self, grads_list, damping): self._input_factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedKroneckerFactor, ((self._inputs,), - self._has_bias)) + fisher_factors.FullyConnectedKroneckerFactor, + ((self._inputs,), self._has_bias)) self._output_factor = self._layer_collection.make_or_get_factor( fisher_factors.FullyConnectedKroneckerFactor, (grads_list,)) self._register_damped_input_and_output_inverses(damping) @@ -438,8 +511,8 @@ class ConvKFCBasicFB(KroneckerProductFB): self._filter_shape = tuple(fltr.shape.as_list()) input_shape = tuple(inputs.shape.as_list()) - self._num_locations = (input_shape[1] * input_shape[2] // - (strides[1] * strides[2])) + self._num_locations = ( + input_shape[1] * input_shape[2] // (strides[1] * strides[2])) super(ConvKFCBasicFB, self).__init__(layer_collection) @@ -461,3 +534,30 @@ class ConvKFCBasicFB(KroneckerProductFB): def tensors_to_compute_grads(self): return self._outputs + + +def _concat_along_batch_dim(tensor_list): + """Concatenate tensors along batch (first) dimension. + + Args: + tensor_list: list of Tensors or list of tuples of Tensors. + + Returns: + Tensor or tuple of Tensors. + + Raises: + ValueError: If 'tensor_list' is empty. + + """ + if not tensor_list: + raise ValueError( + "Cannot concatenate Tensors if there are no Tensors to concatenate.") + + if isinstance(tensor_list[0], (tuple, list)): + # [(tensor1a, tensor1b), + # (tensor2a, tensor2b), ...] --> (tensor_a, tensor_b) + return tuple( + array_ops.concat(tensors, axis=0) for tensors in zip(*tensor_list)) + else: + # [tensor1, tensor2] --> tensor + return array_ops.concat(tensor_list, axis=0) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index d3c783ee2f..86a1782fcf 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -428,11 +428,28 @@ class NaiveDiagonalFactor(DiagonalFactor): class FullyConnectedDiagonalFactor(DiagonalFactor): - """FisherFactor for a diagonal approx of a fully-connected layer's Fisher.""" + r"""FisherFactor for a diagonal approx of a fully-connected layer's Fisher. + + Given in = [batch_size, input_size] and out_grad = [batch_size, output_size], + approximates the covariance as, + + Cov(in, out) = (1/batch_size) \sum_{i} outer(in[i], out_grad[i]) ** 2.0 + + where the square is taken element-wise. + """ # TODO(jamesmartens): add units tests for this class def __init__(self, inputs, outputs_grads, has_bias=False): + """Instantiate FullyConnectedDiagonalFactor. + + Args: + inputs: Tensor of shape [batch_size, input_size]. Inputs to fully + connected layer. + outputs_grads: List of Tensors of shape [batch_size, output_size]. + Gradient of loss with respect to layer's preactivations. + has_bias: bool. If True, append '1' to each input. + """ self._outputs_grads = outputs_grads self._batch_size = array_ops.shape(inputs)[0] self._orig_tensors_name = scope_string_from_params((inputs,) + diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 0cb55894ad..beb8ef136e 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -259,9 +259,9 @@ class LayerCollection(object): fb.FullyConnectedKFACBasicFB(self, inputs, outputs, has_bias)) elif approx == APPROX_DIAGONAL_NAME: - self.register_block(params, - fb.FullyConnectedDiagonalFB(self, inputs, outputs, - has_bias)) + block = fb.FullyConnectedDiagonalFB(self, has_bias) + block.register_additional_minibatch(inputs, outputs) + self.register_block(params, block) else: raise ValueError("Bad value {} for approx.".format(approx)) -- GitLab From f4db3b27430479cccb51518952102d63a3ebc916 Mon Sep 17 00:00:00 2001 From: gunan Date: Wed, 18 Oct 2017 20:47:56 -0700 Subject: [PATCH 138/573] Disable flaky tests in cmake build. (#13816) --- tensorflow/contrib/cmake/tf_tests.cmake | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index 24f21afdfc..3cc874e4f2 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -192,12 +192,10 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py" # generally not working "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py" - # flaky test - "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py" + # Fails because uses data dependencies with bazel "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py" - # flaky tests - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py" # takes very long to run - "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py" + # Takes very long to run without sharding (defined in bazel build file). + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py" # Loading resources in contrib doesn't seem to work on Windows "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py" "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py" @@ -216,14 +214,22 @@ if (tensorflow_BUILD_PYTHON_TESTS) # stl on windows handles overflows different "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py" # Numerical issues, calculations off. "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py" + "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py" # Float division by zero "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py" + # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces. + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/batch_matmul_op_test.py" + # Flaky because of local cluster creation. + "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py" + "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py" + "${tensorflow_source_dir}tensorflow/python/training/localhost_cluster_performance_test.py" + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py" + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py" + "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py" # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU. "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py" @@ -233,6 +239,7 @@ if (tensorflow_BUILD_PYTHON_TESTS) # Depends on gemmlowp -> pthread "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py" # int32/int64 mixup + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py" -- GitLab From 52e9b19b4fdf7ac966858f906c397e5ba2a1bf85 Mon Sep 17 00:00:00 2001 From: Tian Jin Date: Thu, 19 Oct 2017 10:27:40 -0400 Subject: [PATCH 139/573] fix aws build file for linux ppc --- third_party/aws.BUILD | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD index 9d8e7946cd..bc6a2fd8cc 100644 --- a/third_party/aws.BUILD +++ b/third_party/aws.BUILD @@ -18,6 +18,9 @@ cc_library( "@%ws%//tensorflow:darwin": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), + "@%ws%//tensorflow:linux_ppc64le": glob([ + "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", + ]), "//conditions:default": [], }) + glob([ "aws-cpp-sdk-core/include/**/*.h", @@ -57,6 +60,11 @@ cc_library( "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", ], + "@%ws%//tensorflow:linux_ppc64le": [ + "PLATFORM_LINUX", + "ENABLE_CURL_CLIENT", + "ENABLE_NO_ENCRYPTION", + ], "//conditions:default": [], }), includes = [ -- GitLab From 502340916822f8cafade906c1c42acd842ddb7ed Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 19 Oct 2017 09:43:01 -0700 Subject: [PATCH 140/573] tfe.enable_eager_execution fails only if graphs have been created. It is otherwise idempotent. PiperOrigin-RevId: 172757241 --- tensorflow/contrib/eager/python/tfe.py | 4 +- tensorflow/contrib/eager/python/tfe_test.py | 8 --- tensorflow/python/eager/BUILD | 2 +- tensorflow/python/eager/context.py | 64 --------------------- tensorflow/python/eager/test.py | 4 +- tensorflow/python/framework/ops.py | 58 +++++++++++++++++++ 6 files changed, 63 insertions(+), 77 deletions(-) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 1acb1ba1b8..c519df8b5c 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -67,12 +67,10 @@ from tensorflow.contrib.eager.python.saver import Saver from tensorflow.contrib.eager.python.summary_writer import SummaryWriter from tensorflow.python.eager import backprop from tensorflow.python.eager import function -from tensorflow.python.eager.context import enable_eager_execution from tensorflow.python.eager.context import in_eager_mode from tensorflow.python.eager.context import in_graph_mode from tensorflow.python.eager.context import list_devices from tensorflow.python.eager.context import num_gpus -from tensorflow.python.eager.context import run from tensorflow.python.eager.core import enable_tracing from tensorflow.python.eager.custom_gradient import custom_gradient from tensorflow.python.eager.execution_callbacks import add_execution_callback @@ -81,6 +79,8 @@ from tensorflow.python.eager.execution_callbacks import inf_callback from tensorflow.python.eager.execution_callbacks import inf_nan_callback from tensorflow.python.eager.execution_callbacks import nan_callback from tensorflow.python.eager.execution_callbacks import seterr +from tensorflow.python.framework.ops import enable_eager_execution +from tensorflow.python.framework.ops import eager_run as run from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable from tensorflow.python.util.all_util import remove_undocumented diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index eab8958f23..6b5053125b 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -95,14 +95,6 @@ class TFETest(test_util.TensorFlowTestCase): devices = tfe.list_devices() self.assertEqual(len(devices) - 1, tfe.num_gpus()) - def testCallingEnableEagerExecutionMoreThanOnce(self): - # Note that eager.test.main() has already invoked enable_eager_exceution(). - with self.assertRaisesRegexp( - ValueError, r'Do not call tfe\.%s more than once in the same process' % - tfe.enable_eager_execution.__name__): - tfe.enable_eager_execution() - - if __name__ == '__main__': tfe.enable_eager_execution() test.main() diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index ef04f933c5..f5b946ec26 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -151,9 +151,9 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - ":context", ":core", "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", ], ) diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 996748a870..aa7cba56de 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -26,10 +26,8 @@ import threading from tensorflow.python import pywrap_tensorflow from tensorflow.python.framework import device as pydev from tensorflow.python.framework import errors -from tensorflow.python.platform import app from tensorflow.python.util import compat from tensorflow.python.util import tf_contextlib -from tensorflow.python.util import tf_inspect GRAPH_MODE = 0 EAGER_MODE = 1 @@ -423,68 +421,6 @@ def device(name): return context().device(name) -def run(main=None, argv=None): - """Runs the program with an optional main function and argv list. - - The program will run with eager execution enabled. - - Example: - ```python - import tensorflow as tf - # Import subject to future changes: - from tensorflow.contrib.eager.python import tfe - - def main(_): - u = tf.constant(6.0) - v = tf.constant(7.0) - print(u * v) - - if __name__ == "__main__": - tfe.run() - ``` - - Args: - main: the main function to run. - argv: the arguments to pass to it. - """ - enable_eager_execution() - app.run(main, argv) - - -# TODO(apassos): This should not be a part of the public API. -def enable_eager_execution(): - """Enables, for the rest of the lifetime of this program, eager execution. - - If not called immediately on startup risks creating breakage and bugs. Calling - this method more than once in the same process will lead to an exception. - - Example: - ```python - # Before eager execution is enabled, `Tensor`s are symbolic and do not hold - # concrete values (they are to be executed in a `tf.Session`). - assert not hasattr(tf.multiply(6, 7), "numpy") - - tfe.enable_eager_execution() - - # After eager execution is enabled, operations are executed as they are - # defined and `Tensor`s hold concrete values, which can be accessed as - # `numpy.ndarray`s through the `numpy()` method. - assert tf.multiply(6, 7).numpy() == 42 - ``` - - Raises: - ValueError: If this method has already been invoked in the current process. - """ - global _default_mode - if _default_mode == EAGER_MODE: - func_name = ( - "tfe." + tf_inspect.getframeinfo(tf_inspect.currentframe()).function) - raise ValueError( - "Do not call %s more than once in the same process. Note eager-mode " - "methods such as tfe.run() also call %s." % (func_name, func_name)) - _default_mode = EAGER_MODE - - def list_devices(): """List the names of the available devices. diff --git a/tensorflow/python/eager/test.py b/tensorflow/python/eager/test.py index 3d8af7e056..f6a46e7eb3 100644 --- a/tensorflow/python/eager/test.py +++ b/tensorflow/python/eager/test.py @@ -18,11 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.eager import context as _context +from tensorflow.python.framework import ops as _ops from tensorflow.python.platform import test as _test from tensorflow.python.platform.test import * # pylint: disable=wildcard-import def main(argv=None): - _context.enable_eager_execution() + _ops.enable_eager_execution() _test.main(argv) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index e7e36941e5..ef708a4703 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -47,6 +47,7 @@ from tensorflow.python.framework import op_def_registry from tensorflow.python.framework import registry from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import versions +from tensorflow.python.platform import app from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat from tensorflow.python.util import decorator_utils @@ -4558,6 +4559,63 @@ class _DefaultGraphStack(_DefaultStack): # pylint: disable=protected-access _default_graph_stack = _DefaultGraphStack() +def enable_eager_execution(): + """Enables, for the rest of the lifetime of this program, eager execution. + + If not called immediately on startup risks creating breakage and bugs. + + Example: + ```python + tfe.enable_eager_execution() + + # After eager execution is enabled, operations are executed as they are + # defined and `Tensor`s hold concrete values, which can be accessed as + # `numpy.ndarray`s through the `numpy()` method. + assert tf.multiply(6, 7).numpy() == 42 + ``` + + Raises: + ValueError: If this method has already been invoked in the current process. + """ + # pylint: disable=protected-access + if context._default_mode == context.GRAPH_MODE: + graph_mode_has_been_used = ( + _default_session_stack.stack or + _default_graph_stack._global_default_graph is not None) + if graph_mode_has_been_used: + raise ValueError( + "tfe.enable_eager_execution has to be called at program startup.") + context._default_mode = context.EAGER_MODE + + +def eager_run(main=None, argv=None): + """Runs the program with an optional main function and argv list. + + The program will run with eager execution enabled. + + Example: + ```python + import tensorflow as tf + # Import subject to future changes: + from tensorflow.contrib.eager.python import tfe + + def main(_): + u = tf.constant(6.0) + v = tf.constant(7.0) + print(u * v) + + if __name__ == "__main__": + tfe.run() + ``` + + Args: + main: the main function to run. + argv: the arguments to pass to it. + """ + enable_eager_execution() + app.run(main, argv) + + def reset_default_graph(): """Clears the default graph stack and resets the global default graph. -- GitLab From df22cf83a21b62838ecf6f3a1c8a9c30ab20d482 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 09:43:08 -0700 Subject: [PATCH 141/573] Makes export_outputs in multi_label_head consistent with multi_class_head and simplifies multi_head export_outputs. PiperOrigin-RevId: 172757258 --- tensorflow/contrib/estimator/BUILD | 1 + .../estimator/python/estimator/head.py | 11 +++- .../estimator/python/estimator/head_test.py | 27 +++++++- .../estimator/python/estimator/multi_head.py | 5 +- .../python/estimator/multi_head_test.py | 4 +- tensorflow/python/estimator/canned/head.py | 63 +++++++++---------- 6 files changed, 72 insertions(+), 39 deletions(-) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 4dd9f19ec3..89c26d1d2f 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -135,6 +135,7 @@ py_library( "//tensorflow/python/estimator:model_fn", "//tensorflow/python/estimator:prediction_keys", "//tensorflow/python/ops/losses", + "//tensorflow/python/saved_model:signature_constants", ], ) diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index ebf91e8bb4..d01b30d7f9 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -33,8 +33,11 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics as metrics_lib from tensorflow.python.ops import sparse_ops from tensorflow.python.ops.losses import losses +from tensorflow.python.saved_model import signature_constants from tensorflow.python.summary import summary +_DEFAULT_SERVING_KEY = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY + def multi_class_head(n_classes, weight_column=None, @@ -287,11 +290,17 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: + classifier_output = head_lib._classification_output( # pylint:disable=protected-access + scores=probabilities, n_classes=self._n_classes, + label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ - '': export_output.ClassificationOutput(scores=probabilities) + _DEFAULT_SERVING_KEY: classifier_output, + head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access + head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access + export_output.PredictOutput(predictions)) }) # Eval. diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py index ec1386af34..b7252f93ee 100644 --- a/tensorflow/contrib/estimator/python/estimator/head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/head_test.py @@ -143,6 +143,7 @@ class MultiLabelHead(test.TestCase): logits = np.array( [[0., 1., 2., -1.], [-1., -2., -3., 1.]], dtype=np.float32) expected_probabilities = _sigmoid(logits) + expected_export_classes = [[b'0', b'1', b'2', b'3']] * 2 spec = head.create_estimator_spec( features={'x': np.array(((42,),), dtype=np.int32)}, @@ -150,7 +151,8 @@ class MultiLabelHead(test.TestCase): logits=logits) self.assertItemsEqual( - ('', _DEFAULT_SERVING_KEY), spec.export_outputs.keys()) + (_DEFAULT_SERVING_KEY, 'predict', 'classification'), + spec.export_outputs.keys()) # Assert predictions and export_outputs. with self.test_session() as sess: @@ -166,6 +168,29 @@ class MultiLabelHead(test.TestCase): self.assertAllClose( expected_probabilities, sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].scores)) + self.assertAllEqual( + expected_export_classes, + sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].classes)) + + def test_predict_with_label_vocabulary(self): + n_classes = 4 + head = head_lib.multi_label_head( + n_classes, label_vocabulary=['foo', 'bar', 'foobar', 'barfoo']) + + logits = np.array( + [[0., 1., 2., -1.], [-1., -2., -3., 1.]], dtype=np.float32) + expected_export_classes = [[b'foo', b'bar', b'foobar', b'barfoo']] * 2 + + spec = head.create_estimator_spec( + features={'x': np.array(((42,),), dtype=np.int32)}, + mode=model_fn.ModeKeys.PREDICT, + logits=logits) + + with self.test_session() as sess: + _initialize_variables(self, spec.scaffold) + self.assertAllEqual( + expected_export_classes, + sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].classes)) def test_weight_should_not_impact_prediction(self): n_classes = 4 diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head.py b/tensorflow/contrib/estimator/python/estimator/multi_head.py index e6340424f7..64b2a9dee8 100644 --- a/tensorflow/contrib/estimator/python/estimator/multi_head.py +++ b/tensorflow/contrib/estimator/python/estimator/multi_head.py @@ -236,7 +236,10 @@ class _MultiHead(head_lib._Head): # pylint:disable=protected-access for head, spec in zip(self._heads, all_estimator_spec): head_name = head.name for k, v in six.iteritems(spec.export_outputs): - key = '%s/%s' % (k, head_name) if k else head_name + if k == _DEFAULT_SERVING_KEY: + key = head_name + else: + key = '%s/%s' % (k, head_name) export_outputs[key] = v for k, v in six.iteritems(spec.predictions): predictions[(head_name, k)] = v diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py index e86cb2b96f..48027035ce 100644 --- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py @@ -126,8 +126,8 @@ class MultiHeadTest(test.TestCase): logits=logits) self.assertItemsEqual( - (_DEFAULT_SERVING_KEY, _DEFAULT_SERVING_KEY + '/head1', 'head1', - _DEFAULT_SERVING_KEY + '/head2', 'head2'), + (_DEFAULT_SERVING_KEY, 'head1', 'classification/head1', 'predict/head1', + 'head2', 'classification/head2', 'predict/head2'), spec.export_outputs.keys()) # Assert predictions and export_outputs. diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py index beafe0d5c4..1cc82c5055 100644 --- a/tensorflow/python/estimator/canned/head.py +++ b/tensorflow/python/estimator/canned/head.py @@ -269,6 +269,21 @@ def _indicator_labels_mean(labels, weights=None, name=None): return metrics_lib.mean(labels, weights=weights, name=scope) +def _classification_output(scores, n_classes, label_vocabulary=None): + batch_size = array_ops.shape(scores)[0] + if label_vocabulary: + export_class_list = label_vocabulary + else: + export_class_list = string_ops.as_string(math_ops.range(n_classes)) + export_output_classes = array_ops.tile( + input=array_ops.expand_dims(input=export_class_list, axis=0), + multiples=[batch_size, 1]) + return export_output.ClassificationOutput( + scores=scores, + # `ClassificationOutput` requires string classes. + classes=export_output_classes) + + def _accuracy_baseline(labels_mean): """Return accuracy baseline based on labels mean. @@ -401,12 +416,11 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): def logits_dimension(self): return self._n_classes - def _eval_metric_ops(self, labels, probabilities, logits, - class_ids, weights, unweighted_loss): + def _eval_metric_ops(self, labels, class_ids, weights, unweighted_loss): """Returns the Eval metric ops.""" with ops.name_scope( None, 'metrics', - (labels, probabilities, logits, class_ids, weights, unweighted_loss)): + (labels, class_ids, weights, unweighted_loss)): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. @@ -479,18 +493,9 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: - batch_size = array_ops.shape(probabilities)[0] - export_class_list = self._label_vocabulary - if not export_class_list: - export_class_list = string_ops.as_string( - math_ops.range(self._n_classes)) - export_output_classes = array_ops.tile( - input=array_ops.expand_dims(input=export_class_list, axis=0), - multiples=[batch_size, 1]) - classifier_output = export_output.ClassificationOutput( - scores=probabilities, - # `ClassificationOutput` requires string classes. - classes=export_output_classes) + classifier_output = _classification_output( + scores=probabilities, n_classes=self._n_classes, + label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, @@ -513,8 +518,6 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, - probabilities=probabilities, - logits=logits, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) @@ -611,12 +614,12 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): labels, logits, logistic, - scores, class_ids, unweighted_loss, weights=None): - with ops.name_scope(None, 'metrics', (labels, logits, logistic, scores, - class_ids, unweighted_loss, weights)): + with ops.name_scope( + None, 'metrics', + (labels, logits, logistic, class_ids, unweighted_loss, weights)): keys = metric_keys.MetricKeys labels_mean = _indicator_labels_mean( labels=labels, weights=weights, name=keys.LABEL_MEAN) @@ -709,7 +712,8 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') - scores = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES) + probabilities = nn.softmax( + two_class_logits, name=pred_keys.PROBABILITIES) class_ids = array_ops.reshape( math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') if self._label_vocabulary: @@ -722,22 +726,14 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, - pred_keys.PROBABILITIES: scores, + pred_keys.PROBABILITIES: probabilities, pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: - batch_size = array_ops.shape(logistic)[0] - export_class_list = self._label_vocabulary - if not export_class_list: - export_class_list = string_ops.as_string([0, 1]) - export_output_classes = array_ops.tile( - input=array_ops.expand_dims(input=export_class_list, axis=0), - multiples=[batch_size, 1]) - classifier_output = export_output.ClassificationOutput( - scores=scores, - # `ClassificationOutput` requires string classes. - classes=export_output_classes) + classifier_output = _classification_output( + scores=probabilities, n_classes=2, + label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, @@ -764,7 +760,6 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): labels=processed_labels, logits=logits, logistic=logistic, - scores=scores, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) -- GitLab From fa058053e59efc0d60147b40e265392a858bc3cd Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Thu, 19 Oct 2017 09:47:56 -0700 Subject: [PATCH 142/573] Add `quantile` to `tf.distributions.TransformedDistribution`. PiperOrigin-RevId: 172757818 --- .../transformed_distribution_test.py | 12 ++++++++++++ .../ops/conditional_transformed_distribution.py | 16 ++++++++++++++++ .../distributions/transformed_distribution.py | 13 +++++++++++++ 3 files changed, 41 insertions(+) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py index 4001530f66..103d8e1862 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py @@ -116,6 +116,18 @@ class TransformedDistributionTest(test.TestCase): np.log(sp_normal.pdf(2.13) + sp_normal.pdf(-2.13)), abs_normal.log_prob(2.13).eval()) + def testQuantile(self): + with self.test_session() as sess: + logit_normal = self._cls()( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.Sigmoid(), + validate_args=True) + grid = [0., 0.25, 0.5, 0.75, 1.] + q = logit_normal.quantile(grid) + cdf = logit_normal.cdf(q) + cdf_ = sess.run(cdf) + self.assertAllClose(grid, cdf_, rtol=1e-6, atol=0.) + def testCachedSamples(self): exp_forward_only = bs.Exp(event_ndims=0) exp_forward_only._inverse = self._make_unimplemented( diff --git a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py index f1b7bf468e..599c855cda 100644 --- a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py +++ b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py @@ -198,3 +198,19 @@ class ConditionalTransformedDistribution( distribution_kwargs = distribution_kwargs or {} x = self.bijector.inverse(y, **bijector_kwargs) return self.distribution.survival_function(x, **distribution_kwargs) + + @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) + def _quantile(self, value, bijector_kwargs=None, distribution_kwargs=None): + if self._is_maybe_event_override: + raise NotImplementedError("quantile is not implemented when overriding " + "event_shape") + if not self.bijector._is_injective: # pylint: disable=protected-access + raise NotImplementedError("quantile is not implemented when " + "bijector is not injective.") + bijector_kwargs = bijector_kwargs or {} + distribution_kwargs = distribution_kwargs or {} + # x_q is the "qth quantile" of X iff q = P[X <= x_q]. Now, since X = + # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)], + # implies the qth quantile of Y is g(x_q). + inv_cdf = self.distribution.quantile(value, **distribution_kwargs) + return self.bijector.forward(inv_cdf, **bijector_kwargs) diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py index 15a1125f82..ba25b2c348 100644 --- a/tensorflow/python/ops/distributions/transformed_distribution.py +++ b/tensorflow/python/ops/distributions/transformed_distribution.py @@ -503,6 +503,19 @@ class TransformedDistribution(distribution_lib.Distribution): x = self.bijector.inverse(y) return self.distribution.survival_function(x) + def _quantile(self, value): + if self._is_maybe_event_override: + raise NotImplementedError("quantile is not implemented when overriding " + "event_shape") + if not self.bijector._is_injective: # pylint: disable=protected-access + raise NotImplementedError("quantile is not implemented when " + "bijector is not injective.") + # x_q is the "qth quantile" of X iff q = P[X <= x_q]. Now, since X = + # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)], + # implies the qth quantile of Y is g(x_q). + inv_cdf = self.distribution.quantile(value) + return self.bijector.forward(inv_cdf) + def _entropy(self): if not self.bijector.is_constant_jacobian: raise NotImplementedError("entropy is not implemented") -- GitLab From f477d48ce7b89d1ded6c823dcf9518fccd837643 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 19 Oct 2017 10:11:57 -0700 Subject: [PATCH 143/573] [XLA:GPU] Use cuMemcpy for device-to-device data copies PiperOrigin-RevId: 172760878 --- .../compiler/xla/service/gpu/copy_thunk.cc | 28 +++++++++++-- .../compiler/xla/service/gpu/copy_thunk.h | 39 ++++++++++++++---- .../compiler/xla/service/gpu/ir_emitter.h | 8 +++- .../xla/service/gpu/ir_emitter_unnested.cc | 41 ++++++++++++++++--- 4 files changed, 97 insertions(+), 19 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/copy_thunk.cc b/tensorflow/compiler/xla/service/gpu/copy_thunk.cc index 87858e9409..f4498663b1 100644 --- a/tensorflow/compiler/xla/service/gpu/copy_thunk.cc +++ b/tensorflow/compiler/xla/service/gpu/copy_thunk.cc @@ -20,15 +20,16 @@ limitations under the License. namespace xla { namespace gpu { -CopyThunk::CopyThunk(const void* source_address, - const BufferAllocation::Slice& destination_buffer, - uint64 mem_size, const HloInstruction* hlo_instruction) +HostToDeviceCopyThunk::HostToDeviceCopyThunk( + const void* source_address, + const BufferAllocation::Slice& destination_buffer, uint64 mem_size, + const HloInstruction* hlo_instruction) : Thunk(Kind::kCopy, hlo_instruction), source_address_(source_address), destination_buffer_(destination_buffer), mem_size_(mem_size) {} -tensorflow::Status CopyThunk::ExecuteOnStream( +tensorflow::Status HostToDeviceCopyThunk::ExecuteOnStream( const BufferAllocations& buffer_allocations, perftools::gputools::Stream* stream) { perftools::gputools::DeviceMemoryBase destination_data = @@ -37,5 +38,24 @@ tensorflow::Status CopyThunk::ExecuteOnStream( return tensorflow::Status::OK(); } +DeviceToDeviceCopyThunk::DeviceToDeviceCopyThunk( + const BufferAllocation::Slice& source_buffer, + const BufferAllocation::Slice& destination_buffer, uint64 mem_size, + const HloInstruction* hlo_instruction) + : Thunk(Kind::kCopy, hlo_instruction), + source_buffer_(source_buffer), + destination_buffer_(destination_buffer), + mem_size_(mem_size) {} + +tensorflow::Status DeviceToDeviceCopyThunk::ExecuteOnStream( + const BufferAllocations& buffer_allocations, + perftools::gputools::Stream* stream) { + perftools::gputools::DeviceMemoryBase destination_data = + buffer_allocations.GetDeviceAddress(destination_buffer_); + perftools::gputools::DeviceMemoryBase source_data = + buffer_allocations.GetDeviceAddress(source_buffer_); + stream->ThenMemcpy(&destination_data, source_data, mem_size_); + return tensorflow::Status::OK(); +} } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/copy_thunk.h b/tensorflow/compiler/xla/service/gpu/copy_thunk.h index 6b8c432715..e2783fd255 100644 --- a/tensorflow/compiler/xla/service/gpu/copy_thunk.h +++ b/tensorflow/compiler/xla/service/gpu/copy_thunk.h @@ -26,19 +26,18 @@ limitations under the License. namespace xla { namespace gpu { -// A thunk that copies data. For now, it copies data only from host to device. -// But it can be extended to perform device-to-host or intra-device copying. -class CopyThunk : public Thunk { +// A thunk that copies data from a host buffer to a device buffer. +class HostToDeviceCopyThunk : public Thunk { public: // Constructs a CopyThunk that copies host data from `source_address` to the // device buffer `destination_buffer`. `mem_size` is the size of the data in // bytes. - CopyThunk(const void* source_address, - const BufferAllocation::Slice& destination_buffer, uint64 mem_size, - const HloInstruction* hlo_instruction); + HostToDeviceCopyThunk(const void* source_address, + const BufferAllocation::Slice& destination_buffer, + uint64 mem_size, const HloInstruction* hlo_instruction); - CopyThunk(const CopyThunk&) = delete; - CopyThunk& operator=(const CopyThunk&) = delete; + HostToDeviceCopyThunk(const HostToDeviceCopyThunk&) = delete; + HostToDeviceCopyThunk& operator=(const HostToDeviceCopyThunk&) = delete; tensorflow::Status ExecuteOnStream( const BufferAllocations& buffer_allocations, @@ -50,6 +49,30 @@ class CopyThunk : public Thunk { const uint64 mem_size_; }; +// A thunk that copies data from a device buffer to another device buffer. +class DeviceToDeviceCopyThunk : public Thunk { + public: + // Constructs a CopyThunk that copies host data from `source_buffer` to the + // device buffer `destination_buffer`. `mem_size` is the size of the data in + // bytes. + DeviceToDeviceCopyThunk(const BufferAllocation::Slice& source_buffer, + const BufferAllocation::Slice& destination_buffer, + uint64 mem_size, + const HloInstruction* hlo_instruction); + + DeviceToDeviceCopyThunk(const DeviceToDeviceCopyThunk&) = delete; + DeviceToDeviceCopyThunk& operator=(const DeviceToDeviceCopyThunk&) = delete; + + tensorflow::Status ExecuteOnStream( + const BufferAllocations& buffer_allocations, + perftools::gputools::Stream* stream) override; + + private: + const BufferAllocation::Slice source_buffer_; + const BufferAllocation::Slice destination_buffer_; + const uint64 mem_size_; +}; + } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h index 5e3f3bfdf1..29c3761dc5 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h @@ -339,8 +339,12 @@ class IrEmitterUnnested : public IrEmitter { // to make sure `inst` outlives the lifetime of the returned Thunk object. std::unique_ptr BuildGemmThunk(const HloInstruction* inst); - // Returns a CopyThunk that calls host-to-device cuMemcpy to implement `inst`. - std::unique_ptr BuildCopyThunk(const HloInstruction* inst); + // Returns a thunk that calls host-to-device cuMemcpy to implement `inst`. + std::unique_ptr BuildHostToDeviceCopyThunk(const HloInstruction* inst); + + // Returns a thunk that calls device-to-device cuMemcpy to implement `inst`. + std::unique_ptr BuildDeviceToDeviceCopyThunk( + const HloInstruction* inst); // Returns an InfeedThunk that performs device-to-device memcpy to implement // `inst`. diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 120d50ed25..734c793c15 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -146,7 +146,7 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { } namespace { -bool ImplementedAsMemcpy(const HloInstruction& hlo) { +bool ImplementedAsHostToDeviceMemcpy(const HloInstruction& hlo) { // `hlo` needs to satisfy three conditions to be implemented as a // host-to-device cuMemcpy. // @@ -157,6 +157,20 @@ bool ImplementedAsMemcpy(const HloInstruction& hlo) { hlo.operand(0)->opcode() == HloOpcode::kConstant && ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()); } + +bool ImplementedAsDeviceToDeviceMemcpy( + const BufferAssignment& buffer_assignment, const HloInstruction& hlo) { + // `hlo` needs to satisfy three conditions to be implemented as a + // device-to-device cuMemcpy. + // + // 1. `hlo` is a kCopy instruction. + // 2. `hlo` and its operand have the same shape (thus the same layout too). + // 3. The operand to `hlo` has a buffer assignment (constants do not, for + // instance) which means the source buffer also resides on the device. + return hlo.opcode() == HloOpcode::kCopy && + ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()) && + buffer_assignment.HasTopLevelAllocation(hlo.operand(0)); +} } // namespace llvm::Function* IrEmitterUnnested::BuildKernelPrototype( @@ -664,8 +678,13 @@ int64 EmitTranspose021Tiled(llvm_ir::IrArray input, llvm_ir::IrArray output, } // namespace Status IrEmitterUnnested::HandleCopy(HloInstruction* copy) { - if (ImplementedAsMemcpy(*copy)) { - thunk_sequence_->emplace_back(BuildCopyThunk(copy)); + if (ImplementedAsHostToDeviceMemcpy(*copy)) { + thunk_sequence_->emplace_back(BuildHostToDeviceCopyThunk(copy)); + return Status::OK(); + } + if (ImplementedAsDeviceToDeviceMemcpy( + ir_emitter_context_->buffer_assignment(), *copy)) { + thunk_sequence_->emplace_back(BuildDeviceToDeviceCopyThunk(copy)); return Status::OK(); } bool is_transpose_021; @@ -1579,11 +1598,11 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( llvm_ir::AsString(kernel->getName()), inst); } -std::unique_ptr IrEmitterUnnested::BuildCopyThunk( +std::unique_ptr IrEmitterUnnested::BuildHostToDeviceCopyThunk( const HloInstruction* inst) { const HloInstruction* operand = inst->operand(0); CHECK_EQ(HloOpcode::kConstant, operand->opcode()); - return MakeUnique( + return MakeUnique( /*source_address=*/operand->literal().InternalData(), /*destination_buffer=*/GetAllocationSlice(*inst), /*mem_size=*/ @@ -1592,6 +1611,18 @@ std::unique_ptr IrEmitterUnnested::BuildCopyThunk( inst); } +std::unique_ptr IrEmitterUnnested::BuildDeviceToDeviceCopyThunk( + const HloInstruction* inst) { + const HloInstruction* operand = inst->operand(0); + return MakeUnique( + /*source_address=*/GetAllocationSlice(*operand), + /*destination_buffer=*/GetAllocationSlice(*inst), + /*mem_size=*/ + llvm_ir::ByteSizeOf(operand->shape(), + ir_emitter_context_->llvm_module()->getDataLayout()), + inst); +} + std::unique_ptr IrEmitterUnnested::BuildInfeedThunk( const HloInstruction* inst) { CHECK_EQ(HloOpcode::kInfeed, inst->opcode()); -- GitLab From fb7892e6d0d749251415fe308c618667058c1c7c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 10:44:04 -0700 Subject: [PATCH 144/573] Note that tfgan.eval.preprocess_image now expects inputs from [0, 255] range. PiperOrigin-RevId: 172765720 --- .../contrib/gan/python/eval/python/classifier_metrics_impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py index 4af87b8b47..f7c70db0e0 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py @@ -107,7 +107,6 @@ def _symmetric_matrix_square_root(mat, eps=1e-10): # Convenience preprocessing function, with fixed defaults. -# NOTE: Floating-point inputs are expected to be in [0, 1]. # Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py. def preprocess_image( images, height=INCEPTION_DEFAULT_IMAGE_SIZE, -- GitLab From 89f15928293f62b63404ffbdabbb6a3b07274ff8 Mon Sep 17 00:00:00 2001 From: chi-hung Date: Fri, 20 Oct 2017 02:10:26 +0800 Subject: [PATCH 145/573] fix the issue: "libcuda.so.1 not found" and add some minor changes. (#13811) * fix the issue: "libcuda.so.1 not found" and add some minor changes. * Install wheel instead of python-wheel. Also, unnecessary lines are removed. --- .../docker/Dockerfile.devel-gpu-cuda9-cudnn7 | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 index 75351ecfba..4558bc5293 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 @@ -42,6 +42,7 @@ RUN pip --no-cache-dir install \ scipy \ sklearn \ pandas \ + wheel \ && \ python -m ipykernel.kernelspec @@ -80,22 +81,32 @@ RUN git clone https://github.com/tensorflow/tensorflow.git && \ WORKDIR /tensorflow # Configure the build for our CUDA configuration. -ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH -ENV TF_NEED_CUDA 1 -ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1 -ENV TF_CUDA_VERSION 9.0 -ENV TF_CUDNN_VERSION 7 +ENV CI_BUILD_PYTHON=python \ + LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:${LD_LIBRARY_PATH} \ + CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu \ + PYTHON_BIN_PATH=/usr/bin/python \ + PYTHON_LIB_PATH=/usr/local/lib/python2.7/dist-packages \ + TF_NEED_CUDA=1 \ + TF_CUDA_VERSION=9.0 \ + TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1,7.0 \ + TF_CUDNN_VERSION=7 RUN ./configure -RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ - bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ - --jobs=${TF_AVAILABLE_CPUS} \ - tensorflow/tools/pip_package:build_pip_package && \ - mkdir -p /pip_pkg && \ +# Build and Install TensorFlow. +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ + LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ + bazel build -c opt \ + --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + --jobs=${TF_AVAILABLE_CPUS} \ + tensorflow/tools/pip_package:build_pip_package && \ + mkdir /pip_pkg && \ bazel-bin/tensorflow/tools/pip_package/build_pip_package /pip_pkg +# Clean up pip wheel and Bazel cache when done. RUN pip --no-cache-dir install --upgrade /pip_pkg/tensorflow-*.whl && \ + rm -rf /pip_pkg && \ + rm -rf /root/.cache WORKDIR /root -- GitLab From 31587244e4821fbb4eebcf7847281a1df3da6a2a Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Thu, 19 Oct 2017 11:22:37 -0700 Subject: [PATCH 146/573] Remove sklearn from text_classification_cnn.py. PiperOrigin-RevId: 172772457 --- .../examples/learn/text_classification_cnn.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tensorflow/examples/learn/text_classification_cnn.py b/tensorflow/examples/learn/text_classification_cnn.py index 0ee2405c8b..be262285a3 100644 --- a/tensorflow/examples/learn/text_classification_cnn.py +++ b/tensorflow/examples/learn/text_classification_cnn.py @@ -22,7 +22,6 @@ import sys import numpy as np import pandas -from sklearn import metrics import tensorflow as tf FLAGS = None @@ -134,23 +133,15 @@ def main(unused_argv): shuffle=True) classifier.train(input_fn=train_input_fn, steps=100) - # Predict. + # Evaluate. test_input_fn = tf.estimator.inputs.numpy_input_fn( x={WORDS_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) - predictions = classifier.predict(input_fn=test_input_fn) - y_predicted = np.array(list(p['class'] for p in predictions)) - y_predicted = y_predicted.reshape(np.array(y_test).shape) - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - - # Score with tensorflow. scores = classifier.evaluate(input_fn=test_input_fn) - print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) + print('Accuracy: {0:f}'.format(scores['accuracy'])) if __name__ == '__main__': -- GitLab From 1c878e69cbb6276ee38364e42fde4db2239e15a6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 11:39:07 -0700 Subject: [PATCH 147/573] Fixed small typo in a docstring. PiperOrigin-RevId: 172775242 --- tensorflow/python/ops/nn_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 8876591e53..5f82323bfc 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -2114,7 +2114,7 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: di def top_k(input, k=1, sorted=True, name=None): """Finds values and indices of the `k` largest entries for the last dimension. - If the input is a vector (rank-1), finds the `k` largest entries in the vector + If the input is a vector (rank=1), finds the `k` largest entries in the vector and outputs their values and indices as vectors. Thus `values[j]` is the `j`-th largest entry in `input`, and its index is `indices[j]`. -- GitLab From e6e21eb8203183c32dfc156566b58338d5ba204b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 12:20:03 -0700 Subject: [PATCH 148/573] internal change. PiperOrigin-RevId: 172780953 --- .../boosted_trees/lib/utils/batch_features.h | 16 ++++++++++++++++ .../resources/decision_tree_ensemble_resource.h | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h index bb11dc9a07..7a550d6f73 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h @@ -45,6 +45,22 @@ class BatchFeatures { std::vector sparse_int_feature_values_list, std::vector sparse_int_feature_shapes_list); + Status GetFeatureColumnSizes(int64* const num_dense_float_features, + int64* const num_sparse_float_features, + int64* const num_sparse_int_features) const { + QCHECK_NE(num_dense_float_features, nullptr); + QCHECK_NE(num_sparse_float_features, nullptr); + QCHECK_NE(num_sparse_int_features, nullptr); + *num_dense_float_features = dense_float_feature_columns_.size(); + *num_sparse_float_features = sparse_float_feature_columns_.size(); + *num_sparse_int_features = sparse_int_feature_columns_.size(); + if (*num_dense_float_features == 0 && *num_sparse_float_features == 0 && + *num_sparse_int_features == 0) { + return errors::FailedPrecondition("Not intialized yet."); + } + return Status::OK(); + } + // Creates an example iterable for the requested slice. ExamplesIterable examples_iterable(int64 example_start, int64 example_end) const { diff --git a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h index 77e6ecb443..284ad5cdb9 100644 --- a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h +++ b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h @@ -47,6 +47,7 @@ class DecisionTreeEnsembleResource : public StampedResource { int32 num_trees() const { return decision_tree_ensemble_->trees_size(); } bool InitFromSerialized(const string& serialized, const int64 stamp_token) { + CHECK_EQ(stamp(), -1) << "Must Reset before Init."; if (ParseProtoUnlimited(decision_tree_ensemble_, serialized)) { set_stamp(stamp_token); return true; @@ -126,7 +127,7 @@ class DecisionTreeEnsembleResource : public StampedResource { // Resets the resource and frees the protos in arena. // Caller needs to hold the mutex lock while calling this. - void Reset() { + virtual void Reset() { // Reset stamp. set_stamp(-1); -- GitLab From 618e28759b0647f7ef98c70f872f4d6efa8e2002 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 19 Oct 2017 12:28:58 -0700 Subject: [PATCH 149/573] eager: Expose the Network class. PiperOrigin-RevId: 172782028 --- tensorflow/contrib/eager/python/tfe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index c519df8b5c..353b9d2bda 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -43,6 +43,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@seterr @@Iterator +@@Network @@Saver @@SummaryWriter @@restore_variables_on_create @@ -62,6 +63,7 @@ from __future__ import print_function # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import # from tensorflow.contrib.eager.python.datasets import Iterator +from tensorflow.contrib.eager.python.network import Network from tensorflow.contrib.eager.python.saver import restore_variables_on_create from tensorflow.contrib.eager.python.saver import Saver from tensorflow.contrib.eager.python.summary_writer import SummaryWriter -- GitLab From a23abe0c226ec1b99f8bddcb2fba65a46365684b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 12:20:03 -0700 Subject: [PATCH 150/573] internal change. PiperOrigin-RevId: 172780953 --- tensorflow/contrib/eager/python/tfe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 353b9d2bda..c519df8b5c 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -43,7 +43,6 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@seterr @@Iterator -@@Network @@Saver @@SummaryWriter @@restore_variables_on_create @@ -63,7 +62,6 @@ from __future__ import print_function # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import # from tensorflow.contrib.eager.python.datasets import Iterator -from tensorflow.contrib.eager.python.network import Network from tensorflow.contrib.eager.python.saver import restore_variables_on_create from tensorflow.contrib.eager.python.saver import Saver from tensorflow.contrib.eager.python.summary_writer import SummaryWriter -- GitLab From 435995bc4ec221ba31a60bbebc20921a8dbd98bd Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 19 Oct 2017 12:28:58 -0700 Subject: [PATCH 151/573] eager: Expose the Network class. PiperOrigin-RevId: 172782028 --- tensorflow/contrib/eager/python/tfe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index c519df8b5c..353b9d2bda 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -43,6 +43,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@seterr @@Iterator +@@Network @@Saver @@SummaryWriter @@restore_variables_on_create @@ -62,6 +63,7 @@ from __future__ import print_function # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import # from tensorflow.contrib.eager.python.datasets import Iterator +from tensorflow.contrib.eager.python.network import Network from tensorflow.contrib.eager.python.saver import restore_variables_on_create from tensorflow.contrib.eager.python.saver import Saver from tensorflow.contrib.eager.python.summary_writer import SummaryWriter -- GitLab From 3f063271c5a862e8c5deeb5a3738ca20db7771d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 12:33:22 -0700 Subject: [PATCH 152/573] Improve inception score documentation. PiperOrigin-RevId: 172782572 --- .../gan/python/eval/python/classifier_metrics_impl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py index f7c70db0e0..d4c080cab3 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py @@ -106,8 +106,6 @@ def _symmetric_matrix_square_root(mat, eps=1e-10): math_ops.matmul(u, array_ops.diag(si)), v, transpose_b=True) -# Convenience preprocessing function, with fixed defaults. -# Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py. def preprocess_image( images, height=INCEPTION_DEFAULT_IMAGE_SIZE, width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None): @@ -116,6 +114,9 @@ def preprocess_image( This is the preprocessing portion of the graph from http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz. + Note that it expects Tensors in [0, 255]. This function maps pixel values to + [-1, 1] and resizes to match the InceptionV1 network. + Args: images: 3-D or 4-D Tensor of images. Values are in [0, 255]. height: Integer. Height of resized output image. -- GitLab From 013125dc926bafab43db507d18f255cd25503f11 Mon Sep 17 00:00:00 2001 From: Kay Zhu Date: Thu, 19 Oct 2017 12:46:01 -0700 Subject: [PATCH 153/573] [XLA] Documentation: use more correct term for fractionally strided convolutions / LHS dilation. PiperOrigin-RevId: 172784169 --- tensorflow/docs_src/performance/xla/operation_semantics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md index 52258cbae7..91c0d5b8c6 100644 --- a/tensorflow/docs_src/performance/xla/operation_semantics.md +++ b/tensorflow/docs_src/performance/xla/operation_semantics.md @@ -328,9 +328,9 @@ placed between each of the entries in that dimension, increasing the size of the array. The holes are filled with a no-op value, which for convolution means zeroes. -Dilation of the rhs is also called atrous convolution. For more details, see the -@{tf.nn.atrous_conv2d}. Dilation of the lhs is -also called deconvolution. +Dilation of the rhs is also called atrous convolution. For more details, see +@{tf.nn.atrous_conv2d}. Dilation of the lhs is also called transposed +convolution. For more details, see @{tf.nn.conv2d_transpose}. The output shape has these dimensions, in this order: -- GitLab From f843da7cb24501f609d3ce46e7279ddd8c34e338 Mon Sep 17 00:00:00 2001 From: David Soergel Date: Thu, 19 Oct 2017 13:11:53 -0700 Subject: [PATCH 154/573] Update serving_input_fn argument name to serving_input_receiver_fn PiperOrigin-RevId: 172787460 --- tensorflow/python/estimator/exporter.py | 28 ++++++++++--------- tensorflow/python/estimator/exporter_test.py | 20 ++++++------- tensorflow/python/estimator/training_test.py | 2 +- ...tensorflow.estimator.-final-exporter.pbtxt | 2 +- ...ensorflow.estimator.-latest-exporter.pbtxt | 2 +- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py index 56400ab935..c6f20d4a9e 100644 --- a/tensorflow/python/estimator/exporter.py +++ b/tensorflow/python/estimator/exporter.py @@ -71,7 +71,7 @@ class _SavedModelExporter(Exporter): def __init__(self, name, - serving_input_fn, + serving_input_receiver_fn, assets_extra=None, as_text=False): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -79,8 +79,8 @@ class _SavedModelExporter(Exporter): Args: name: unique name of this `Exporter` that is going to be used in the export path. - serving_input_fn: a function that takes no arguments and returns an - `ServingInputReceiver`. + serving_input_receiver_fn: a function that takes no arguments and returns + a `ServingInputReceiver`. assets_extra: An optional dict specifying how to populate the assets.extra directory within the exported SavedModel. Each key should give the destination path (including the filename) relative to the assets.extra @@ -95,7 +95,7 @@ class _SavedModelExporter(Exporter): ValueError: if any arguments is invalid. """ self._name = name - self._serving_input_fn = serving_input_fn + self._serving_input_receiver_fn = serving_input_receiver_fn self._assets_extra = assets_extra self._as_text = as_text @@ -109,7 +109,7 @@ class _SavedModelExporter(Exporter): export_result = estimator.export_savedmodel( export_path, - self._serving_input_fn, + self._serving_input_receiver_fn, assets_extra=self._assets_extra, as_text=self._as_text, checkpoint_path=checkpoint_path) @@ -125,7 +125,7 @@ class FinalExporter(Exporter): def __init__(self, name, - serving_input_fn, + serving_input_receiver_fn, assets_extra=None, as_text=False): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -133,8 +133,8 @@ class FinalExporter(Exporter): Args: name: unique name of this `Exporter` that is going to be used in the export path. - serving_input_fn: a function that takes no arguments and returns an - `ServingInputReceiver`. + serving_input_receiver_fn: a function that takes no arguments and returns + a `ServingInputReceiver`. assets_extra: An optional dict specifying how to populate the assets.extra directory within the exported SavedModel. Each key should give the destination path (including the filename) relative to the assets.extra @@ -148,7 +148,8 @@ class FinalExporter(Exporter): Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn, + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, assets_extra, as_text) @property @@ -175,7 +176,7 @@ class LatestExporter(Exporter): def __init__(self, name, - serving_input_fn, + serving_input_receiver_fn, assets_extra=None, as_text=False, exports_to_keep=5): @@ -184,8 +185,8 @@ class LatestExporter(Exporter): Args: name: unique name of this `Exporter` that is going to be used in the export path. - serving_input_fn: a function that takes no arguments and returns an - `ServingInputReceiver`. + serving_input_receiver_fn: a function that takes no arguments and returns + a `ServingInputReceiver`. assets_extra: An optional dict specifying how to populate the assets.extra directory within the exported SavedModel. Each key should give the destination path (including the filename) relative to the assets.extra @@ -202,7 +203,8 @@ class LatestExporter(Exporter): Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn, + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, assets_extra, as_text) self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py index f90c35dce7..8e0f66cece 100644 --- a/tensorflow/python/estimator/exporter_test.py +++ b/tensorflow/python/estimator/exporter_test.py @@ -33,19 +33,19 @@ from tensorflow.python.util import compat class LatestExporterTest(test.TestCase): def test_error_out_if_exports_to_keep_is_zero(self): - def _serving_input_fn(): + def _serving_input_receiver_fn(): pass with self.assertRaisesRegexp(ValueError, "positive number"): exporter = exporter_lib.LatestExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, exports_to_keep=0) self.assertEqual("latest_exporter", exporter.name) def test_latest_exporter(self): - def _serving_input_fn(): + def _serving_input_receiver_fn(): pass export_dir_base = tempfile.mkdtemp() + "export/" @@ -53,7 +53,7 @@ class LatestExporterTest(test.TestCase): exporter = exporter_lib.LatestExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False, exports_to_keep=5) @@ -66,14 +66,14 @@ class LatestExporterTest(test.TestCase): self.assertEqual("export_result_path", export_result) estimator.export_savedmodel.assert_called_with( export_dir_base, - _serving_input_fn, + _serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False, checkpoint_path="checkpoint_path") def test_only_the_last_export_is_saved(self): - def _serving_input_fn(): + def _serving_input_receiver_fn(): pass export_dir_base = tempfile.mkdtemp() + "export/" @@ -81,7 +81,7 @@ class LatestExporterTest(test.TestCase): exporter = exporter_lib.FinalExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False) estimator = test.mock.Mock(spec=estimator_lib.Estimator) @@ -99,7 +99,7 @@ class LatestExporterTest(test.TestCase): self.assertEqual("export_result_path", export_result) estimator.export_savedmodel.assert_called_with( export_dir_base, - _serving_input_fn, + _serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False, checkpoint_path="checkpoint_path") @@ -117,12 +117,12 @@ class LatestExporterTest(test.TestCase): self.assertTrue(gfile.Exists(export_dir_3)) self.assertTrue(gfile.Exists(export_dir_4)) - def _serving_input_fn(): + def _serving_input_receiver_fn(): return array_ops.constant([1]), None exporter = exporter_lib.LatestExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, exports_to_keep=2) estimator = test.mock.Mock(spec=estimator_lib.Estimator) # Garbage collect all but the most recent 2 exports, diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py index d88ca2c925..1862e325e2 100644 --- a/tensorflow/python/estimator/training_test.py +++ b/tensorflow/python/estimator/training_test.py @@ -1569,7 +1569,7 @@ class TrainAndEvaluateIntegrationTest(test.TestCase): serving_input_receiver_fn = ( export_lib.build_parsing_serving_input_receiver_fn(feature_spec)) return exporter_lib.LatestExporter( - name, serving_input_fn=serving_input_receiver_fn) + name, serving_input_receiver_fn=serving_input_receiver_fn) def _extract_loss_and_global_step(self, event_folder): """Returns the loss and global step in last event.""" diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt index 4c2dbc4d37..ee37b1fa21 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt @@ -9,7 +9,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " + argspec: "args=[\'self\', \'name\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " } member_method { name: "export" diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt index ae1483bf3f..2a9d029029 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt @@ -9,7 +9,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], " + argspec: "args=[\'self\', \'name\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], " } member_method { name: "export" -- GitLab From 3472340ecfba0b960e948fcbaed42d1c08d87b6b Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Thu, 19 Oct 2017 13:39:41 -0700 Subject: [PATCH 155/573] Set evaluation_master to master if not set. The current default confuses a lot of users. Delete the tf_random_seed default since it was updated to None in core in cl/172519268. PiperOrigin-RevId: 172791174 --- tensorflow/contrib/tpu/python/tpu/tpu_config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py index 79fd8b839b..3965c087a1 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py @@ -78,22 +78,24 @@ class TPUConfig( class RunConfig(run_config_lib.RunConfig): """RunConfig with TPU support.""" - def __init__(self, tpu_config=None, evaluation_master='', master='', - tf_random_seed=None, **kwargs): + def __init__(self, tpu_config=None, evaluation_master=None, master='', + **kwargs): """Constructs a RunConfig. Args: tpu_config: the TPUConfig that specifies TPU-specific configuration. evaluation_master: a string. The address of the master to use for eval. + Defaults to master if not set. master: a string. The address of the master to use for training. tf_random_seed: an int. Sets the TensorFlow random seed. Defaults to None, which initializes it randomly based on the environment. """ - # We change the default random seed to None because that's a better default. - kwargs['tf_random_seed'] = tf_random_seed super(RunConfig, self).__init__(**kwargs) self._tpu_config = tpu_config or TPUConfig() - self._evaluation_master = evaluation_master + if evaluation_master is None: + self._evaluation_master = master + else: + self._evaluation_master = evaluation_master self._master = master @property -- GitLab From f2aa6c0777369700a9dd79a0c22d7f3f7dcb0835 Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Thu, 19 Oct 2017 13:49:21 -0700 Subject: [PATCH 156/573] Log initialization and warmup time to proto results in benchmark tool. PiperOrigin-RevId: 172792563 --- tensorflow/tools/benchmark/benchmark_model.cc | 60 ++++++++++++++----- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc index f84ae5c7ce..2d59299da4 100644 --- a/tensorflow/tools/benchmark/benchmark_model.cc +++ b/tensorflow/tools/benchmark/benchmark_model.cc @@ -230,6 +230,23 @@ Status CalculateFlops(const GraphDef& graph, return Status::OK(); } +void RecordBenchmarkEntry(const string& output_prefix, + const string& benchmark_name, const string& postfix, + int num_runs, double total_time_s, + double throughput = -1.0) { + std::stringstream stream; + stream << benchmark_name; + if (!postfix.empty()) { + stream << "_" << postfix; + } + + TestReporter node_reporter(output_prefix, stream.str()); + TF_QCHECK_OK(node_reporter.Initialize()); + TF_QCHECK_OK( + node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput)); + TF_QCHECK_OK(node_reporter.Close()); +} + Status RunBenchmark(const std::vector& inputs, const std::vector& outputs, Session* session, StatSummarizer* stats, int64* inference_time_us) { @@ -350,7 +367,7 @@ int Main(int argc, char** argv) { bool show_type = true; bool show_summary = true; bool show_flops = false; - int warmup_runs = 2; + int warmup_runs = 1; std::vector flag_list = { Flag("graph", &graph, "graph file name"), @@ -441,8 +458,14 @@ int Main(int argc, char** argv) { std::unique_ptr session; std::unique_ptr stats; std::unique_ptr graph_def; + + int64 initialization_start_us = Env::Default()->NowMicros(); Status initialize_status = InitializeSession(num_threads, graph, &session, &graph_def); + int64 initialization_end_us = Env::Default()->NowMicros(); + double initialization_time_s = + (initialization_end_us - initialization_start_us) / 1000000.0; + LOG(INFO) << "Initialized session in " << initialization_time_s << "s"; if (!initialize_status.ok()) { return -1; } @@ -587,11 +610,23 @@ int Main(int argc, char** argv) { static_cast(no_stat_wall_time) / (1024 * 1024); // Report the stats. - TestReporter reporter(output_prefix, benchmark_name); - TF_QCHECK_OK(reporter.Initialize()); - TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time, - throughput)); - TF_QCHECK_OK(reporter.Close()); + RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs, + no_stat_wall_time, throughput); + + // Session initialization time. + RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1, + initialization_time_s); + + // First inference time. Note: if warmup_runs is > 1 this will actually be + // an average of all the warmup runs. + RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference", + warmup_runs, warmup_time_us / 1000000.0); + + // Time from starting to intialize TF to getting the first result back. + // This also assumes that only one warmup run is performed. + RecordBenchmarkEntry( + output_prefix, benchmark_name, "meta-init-plus-first-inference", 1, + initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs); std::map node_type_map_count; std::map node_type_map_time; @@ -603,17 +638,10 @@ int Main(int argc, char** argv) { &node_type_map_memory, &node_type_map_times_called, &accumulated_us); for (const auto& time : node_type_map_time) { - std::stringstream stream; - stream << benchmark_name << "_" << time.first; - TestReporter node_reporter(output_prefix, stream.str()); - LOG(INFO) << "Outputting: [" << time.first << "]"; - - TF_QCHECK_OK(node_reporter.Initialize()); - TF_QCHECK_OK(node_reporter.Benchmark( - stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f, - -1.0)); - TF_QCHECK_OK(node_reporter.Close()); + RecordBenchmarkEntry(output_prefix, benchmark_name, time.first, + stat_num_runs, + (time.second * stat_num_runs) / 1000000.0f); } } -- GitLab From 355ec38d80b14353e52b8de9f9db276e18f53e13 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Thu, 19 Oct 2017 13:39:41 -0700 Subject: [PATCH 157/573] Set evaluation_master to master if not set. The current default confuses a lot of users. Delete the tf_random_seed default since it was updated to None in core in cl/172519268. PiperOrigin-RevId: 172791174 --- tensorflow/tools/benchmark/benchmark_model.cc | 60 +++++-------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc index 2d59299da4..f84ae5c7ce 100644 --- a/tensorflow/tools/benchmark/benchmark_model.cc +++ b/tensorflow/tools/benchmark/benchmark_model.cc @@ -230,23 +230,6 @@ Status CalculateFlops(const GraphDef& graph, return Status::OK(); } -void RecordBenchmarkEntry(const string& output_prefix, - const string& benchmark_name, const string& postfix, - int num_runs, double total_time_s, - double throughput = -1.0) { - std::stringstream stream; - stream << benchmark_name; - if (!postfix.empty()) { - stream << "_" << postfix; - } - - TestReporter node_reporter(output_prefix, stream.str()); - TF_QCHECK_OK(node_reporter.Initialize()); - TF_QCHECK_OK( - node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput)); - TF_QCHECK_OK(node_reporter.Close()); -} - Status RunBenchmark(const std::vector& inputs, const std::vector& outputs, Session* session, StatSummarizer* stats, int64* inference_time_us) { @@ -367,7 +350,7 @@ int Main(int argc, char** argv) { bool show_type = true; bool show_summary = true; bool show_flops = false; - int warmup_runs = 1; + int warmup_runs = 2; std::vector flag_list = { Flag("graph", &graph, "graph file name"), @@ -458,14 +441,8 @@ int Main(int argc, char** argv) { std::unique_ptr session; std::unique_ptr stats; std::unique_ptr graph_def; - - int64 initialization_start_us = Env::Default()->NowMicros(); Status initialize_status = InitializeSession(num_threads, graph, &session, &graph_def); - int64 initialization_end_us = Env::Default()->NowMicros(); - double initialization_time_s = - (initialization_end_us - initialization_start_us) / 1000000.0; - LOG(INFO) << "Initialized session in " << initialization_time_s << "s"; if (!initialize_status.ok()) { return -1; } @@ -610,23 +587,11 @@ int Main(int argc, char** argv) { static_cast(no_stat_wall_time) / (1024 * 1024); // Report the stats. - RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs, - no_stat_wall_time, throughput); - - // Session initialization time. - RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1, - initialization_time_s); - - // First inference time. Note: if warmup_runs is > 1 this will actually be - // an average of all the warmup runs. - RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference", - warmup_runs, warmup_time_us / 1000000.0); - - // Time from starting to intialize TF to getting the first result back. - // This also assumes that only one warmup run is performed. - RecordBenchmarkEntry( - output_prefix, benchmark_name, "meta-init-plus-first-inference", 1, - initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs); + TestReporter reporter(output_prefix, benchmark_name); + TF_QCHECK_OK(reporter.Initialize()); + TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time, + throughput)); + TF_QCHECK_OK(reporter.Close()); std::map node_type_map_count; std::map node_type_map_time; @@ -638,10 +603,17 @@ int Main(int argc, char** argv) { &node_type_map_memory, &node_type_map_times_called, &accumulated_us); for (const auto& time : node_type_map_time) { + std::stringstream stream; + stream << benchmark_name << "_" << time.first; + TestReporter node_reporter(output_prefix, stream.str()); + LOG(INFO) << "Outputting: [" << time.first << "]"; - RecordBenchmarkEntry(output_prefix, benchmark_name, time.first, - stat_num_runs, - (time.second * stat_num_runs) / 1000000.0f); + + TF_QCHECK_OK(node_reporter.Initialize()); + TF_QCHECK_OK(node_reporter.Benchmark( + stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f, + -1.0)); + TF_QCHECK_OK(node_reporter.Close()); } } -- GitLab From 70b8b08ae3923cdf3f238c9636a4166e71b0102e Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Thu, 19 Oct 2017 13:49:21 -0700 Subject: [PATCH 158/573] Log initialization and warmup time to proto results in benchmark tool. PiperOrigin-RevId: 172792563 --- tensorflow/tools/benchmark/benchmark_model.cc | 60 ++++++++++++++----- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc index f84ae5c7ce..2d59299da4 100644 --- a/tensorflow/tools/benchmark/benchmark_model.cc +++ b/tensorflow/tools/benchmark/benchmark_model.cc @@ -230,6 +230,23 @@ Status CalculateFlops(const GraphDef& graph, return Status::OK(); } +void RecordBenchmarkEntry(const string& output_prefix, + const string& benchmark_name, const string& postfix, + int num_runs, double total_time_s, + double throughput = -1.0) { + std::stringstream stream; + stream << benchmark_name; + if (!postfix.empty()) { + stream << "_" << postfix; + } + + TestReporter node_reporter(output_prefix, stream.str()); + TF_QCHECK_OK(node_reporter.Initialize()); + TF_QCHECK_OK( + node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput)); + TF_QCHECK_OK(node_reporter.Close()); +} + Status RunBenchmark(const std::vector& inputs, const std::vector& outputs, Session* session, StatSummarizer* stats, int64* inference_time_us) { @@ -350,7 +367,7 @@ int Main(int argc, char** argv) { bool show_type = true; bool show_summary = true; bool show_flops = false; - int warmup_runs = 2; + int warmup_runs = 1; std::vector flag_list = { Flag("graph", &graph, "graph file name"), @@ -441,8 +458,14 @@ int Main(int argc, char** argv) { std::unique_ptr session; std::unique_ptr stats; std::unique_ptr graph_def; + + int64 initialization_start_us = Env::Default()->NowMicros(); Status initialize_status = InitializeSession(num_threads, graph, &session, &graph_def); + int64 initialization_end_us = Env::Default()->NowMicros(); + double initialization_time_s = + (initialization_end_us - initialization_start_us) / 1000000.0; + LOG(INFO) << "Initialized session in " << initialization_time_s << "s"; if (!initialize_status.ok()) { return -1; } @@ -587,11 +610,23 @@ int Main(int argc, char** argv) { static_cast(no_stat_wall_time) / (1024 * 1024); // Report the stats. - TestReporter reporter(output_prefix, benchmark_name); - TF_QCHECK_OK(reporter.Initialize()); - TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time, - throughput)); - TF_QCHECK_OK(reporter.Close()); + RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs, + no_stat_wall_time, throughput); + + // Session initialization time. + RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1, + initialization_time_s); + + // First inference time. Note: if warmup_runs is > 1 this will actually be + // an average of all the warmup runs. + RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference", + warmup_runs, warmup_time_us / 1000000.0); + + // Time from starting to intialize TF to getting the first result back. + // This also assumes that only one warmup run is performed. + RecordBenchmarkEntry( + output_prefix, benchmark_name, "meta-init-plus-first-inference", 1, + initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs); std::map node_type_map_count; std::map node_type_map_time; @@ -603,17 +638,10 @@ int Main(int argc, char** argv) { &node_type_map_memory, &node_type_map_times_called, &accumulated_us); for (const auto& time : node_type_map_time) { - std::stringstream stream; - stream << benchmark_name << "_" << time.first; - TestReporter node_reporter(output_prefix, stream.str()); - LOG(INFO) << "Outputting: [" << time.first << "]"; - - TF_QCHECK_OK(node_reporter.Initialize()); - TF_QCHECK_OK(node_reporter.Benchmark( - stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f, - -1.0)); - TF_QCHECK_OK(node_reporter.Close()); + RecordBenchmarkEntry(output_prefix, benchmark_name, time.first, + stat_num_runs, + (time.second * stat_num_runs) / 1000000.0f); } } -- GitLab From 3238fab55f4e9daf5a06fc44e78082da42fad8a0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 13:50:28 -0700 Subject: [PATCH 159/573] Fix potential Tensor memory leak in GraphCompiler. PiperOrigin-RevId: 172792724 --- tensorflow/compiler/tf2xla/graph_compiler.cc | 29 ++++++++++++++------ tensorflow/compiler/tf2xla/graph_compiler.h | 17 ------------ 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc index 9893afa7a0..8062f0c03c 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.cc +++ b/tensorflow/compiler/tf2xla/graph_compiler.cc @@ -38,6 +38,7 @@ limitations under the License. #include "tensorflow/core/graph/algorithm.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/public/version.h" @@ -84,9 +85,20 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph, } } // namespace Status GraphCompiler::Compile() { - OutputRegistry output_registry(graph_->num_node_ids()); - std::vector topo_sorted_nodes; + // Maintain a mapping from node id to node outputs. + using NodeOutputs = std::vector; + std::vector output_registry(graph_->num_node_ids()); + auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] { + for (const NodeOutputs& outputs : output_registry) { + for (const TensorValue& value : outputs) { + CHECK(!value.is_ref()); + delete value.tensor; + } + } + }); + // XLA requires determinism, generate a stable ordering from DFS. + std::vector topo_sorted_nodes; GetReversePostOrder(*graph_, &topo_sorted_nodes, /*stable_comparator=*/NodeComparatorName()); @@ -94,7 +106,6 @@ Status GraphCompiler::Compile() { PartiallySetupParams(¶ms); for (Node* n : topo_sorted_nodes) { - NodeOutputs node_outputs; OpKernel* op_kernel_raw = nullptr; Status s = flib_->CreateKernel(n->def(), &op_kernel_raw); // Transfer ownership of the kernel to a local smart pointer. @@ -122,9 +133,9 @@ Status GraphCompiler::Compile() { if (e->IsControlEdge()) continue; Node* src = e->src(); TF_RET_CHECK(src->id() < output_registry.size()); - const NodeOutputs& outputs = output_registry[src->id()]; + const NodeOutputs& src_outputs = output_registry[src->id()]; - tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()]; + tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()]; } OpKernelContext op_context(¶ms, n->num_outputs()); @@ -138,15 +149,15 @@ Status GraphCompiler::Compile() { // Set up outputs. Also check if outputs from the previous computation is // valid. + NodeOutputs& outputs = output_registry[n->id()]; + outputs.resize(n->num_outputs()); for (int o = 0; o < n->num_outputs(); ++o) { - const auto tensor_val = op_context.release_output(o); - if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) { + outputs[o] = op_context.release_output(o); + if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) { return errors::Internal("Missing xla_context ", o, "-th output from ", (*op_context.is_output_dead() ? "(dead)" : ""), SummarizeNode(*n)); } - // Set up outputs - output_registry[n->id()].values.push_back(tensor_val); } } return Status::OK(); diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h index 33781d2c21..ba00160b6d 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.h +++ b/tensorflow/compiler/tf2xla/graph_compiler.h @@ -69,23 +69,6 @@ class GraphCompiler { Status Compile(); private: - // NodeOutputs is a wrapper over TensorValues that represents outputs of a - // node. - struct NodeOutputs { - ~NodeOutputs() { - for (auto& v : values) { - CHECK(!v.is_ref()); - delete v.tensor; - } - } - - // Output values of this node. - std::vector values; - }; - - // A mapping from node id to node output. - using OutputRegistry = std::vector; - // Partially sets params. This partially set params can be reused // across multple nodes visit. void PartiallySetupParams(OpKernelContext::Params* params); -- GitLab From 3f56b1402409ad4efb8dd931d5b1b7bdc713597e Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Thu, 19 Oct 2017 13:49:21 -0700 Subject: [PATCH 160/573] Log initialization and warmup time to proto results in benchmark tool. PiperOrigin-RevId: 172792563 --- tensorflow/compiler/tf2xla/graph_compiler.cc | 29 ++++++-------------- tensorflow/compiler/tf2xla/graph_compiler.h | 17 ++++++++++++ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc index 8062f0c03c..9893afa7a0 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.cc +++ b/tensorflow/compiler/tf2xla/graph_compiler.cc @@ -38,7 +38,6 @@ limitations under the License. #include "tensorflow/core/graph/algorithm.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" -#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/public/version.h" @@ -85,20 +84,9 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph, } } // namespace Status GraphCompiler::Compile() { - // Maintain a mapping from node id to node outputs. - using NodeOutputs = std::vector; - std::vector output_registry(graph_->num_node_ids()); - auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] { - for (const NodeOutputs& outputs : output_registry) { - for (const TensorValue& value : outputs) { - CHECK(!value.is_ref()); - delete value.tensor; - } - } - }); - - // XLA requires determinism, generate a stable ordering from DFS. + OutputRegistry output_registry(graph_->num_node_ids()); std::vector topo_sorted_nodes; + // XLA requires determinism, generate a stable ordering from DFS. GetReversePostOrder(*graph_, &topo_sorted_nodes, /*stable_comparator=*/NodeComparatorName()); @@ -106,6 +94,7 @@ Status GraphCompiler::Compile() { PartiallySetupParams(¶ms); for (Node* n : topo_sorted_nodes) { + NodeOutputs node_outputs; OpKernel* op_kernel_raw = nullptr; Status s = flib_->CreateKernel(n->def(), &op_kernel_raw); // Transfer ownership of the kernel to a local smart pointer. @@ -133,9 +122,9 @@ Status GraphCompiler::Compile() { if (e->IsControlEdge()) continue; Node* src = e->src(); TF_RET_CHECK(src->id() < output_registry.size()); - const NodeOutputs& src_outputs = output_registry[src->id()]; + const NodeOutputs& outputs = output_registry[src->id()]; - tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()]; + tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()]; } OpKernelContext op_context(¶ms, n->num_outputs()); @@ -149,15 +138,15 @@ Status GraphCompiler::Compile() { // Set up outputs. Also check if outputs from the previous computation is // valid. - NodeOutputs& outputs = output_registry[n->id()]; - outputs.resize(n->num_outputs()); for (int o = 0; o < n->num_outputs(); ++o) { - outputs[o] = op_context.release_output(o); - if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) { + const auto tensor_val = op_context.release_output(o); + if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) { return errors::Internal("Missing xla_context ", o, "-th output from ", (*op_context.is_output_dead() ? "(dead)" : ""), SummarizeNode(*n)); } + // Set up outputs + output_registry[n->id()].values.push_back(tensor_val); } } return Status::OK(); diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h index ba00160b6d..33781d2c21 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.h +++ b/tensorflow/compiler/tf2xla/graph_compiler.h @@ -69,6 +69,23 @@ class GraphCompiler { Status Compile(); private: + // NodeOutputs is a wrapper over TensorValues that represents outputs of a + // node. + struct NodeOutputs { + ~NodeOutputs() { + for (auto& v : values) { + CHECK(!v.is_ref()); + delete v.tensor; + } + } + + // Output values of this node. + std::vector values; + }; + + // A mapping from node id to node output. + using OutputRegistry = std::vector; + // Partially sets params. This partially set params can be reused // across multple nodes visit. void PartiallySetupParams(OpKernelContext::Params* params); -- GitLab From e858968dc720f47dd15ed8c2d7a5c3910a7e29b1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 13:50:28 -0700 Subject: [PATCH 161/573] Fix potential Tensor memory leak in GraphCompiler. PiperOrigin-RevId: 172792724 --- tensorflow/compiler/tf2xla/graph_compiler.cc | 29 ++++++++++++++------ tensorflow/compiler/tf2xla/graph_compiler.h | 17 ------------ 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc index 9893afa7a0..8062f0c03c 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.cc +++ b/tensorflow/compiler/tf2xla/graph_compiler.cc @@ -38,6 +38,7 @@ limitations under the License. #include "tensorflow/core/graph/algorithm.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/public/version.h" @@ -84,9 +85,20 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph, } } // namespace Status GraphCompiler::Compile() { - OutputRegistry output_registry(graph_->num_node_ids()); - std::vector topo_sorted_nodes; + // Maintain a mapping from node id to node outputs. + using NodeOutputs = std::vector; + std::vector output_registry(graph_->num_node_ids()); + auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] { + for (const NodeOutputs& outputs : output_registry) { + for (const TensorValue& value : outputs) { + CHECK(!value.is_ref()); + delete value.tensor; + } + } + }); + // XLA requires determinism, generate a stable ordering from DFS. + std::vector topo_sorted_nodes; GetReversePostOrder(*graph_, &topo_sorted_nodes, /*stable_comparator=*/NodeComparatorName()); @@ -94,7 +106,6 @@ Status GraphCompiler::Compile() { PartiallySetupParams(¶ms); for (Node* n : topo_sorted_nodes) { - NodeOutputs node_outputs; OpKernel* op_kernel_raw = nullptr; Status s = flib_->CreateKernel(n->def(), &op_kernel_raw); // Transfer ownership of the kernel to a local smart pointer. @@ -122,9 +133,9 @@ Status GraphCompiler::Compile() { if (e->IsControlEdge()) continue; Node* src = e->src(); TF_RET_CHECK(src->id() < output_registry.size()); - const NodeOutputs& outputs = output_registry[src->id()]; + const NodeOutputs& src_outputs = output_registry[src->id()]; - tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()]; + tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()]; } OpKernelContext op_context(¶ms, n->num_outputs()); @@ -138,15 +149,15 @@ Status GraphCompiler::Compile() { // Set up outputs. Also check if outputs from the previous computation is // valid. + NodeOutputs& outputs = output_registry[n->id()]; + outputs.resize(n->num_outputs()); for (int o = 0; o < n->num_outputs(); ++o) { - const auto tensor_val = op_context.release_output(o); - if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) { + outputs[o] = op_context.release_output(o); + if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) { return errors::Internal("Missing xla_context ", o, "-th output from ", (*op_context.is_output_dead() ? "(dead)" : ""), SummarizeNode(*n)); } - // Set up outputs - output_registry[n->id()].values.push_back(tensor_val); } } return Status::OK(); diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h index 33781d2c21..ba00160b6d 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.h +++ b/tensorflow/compiler/tf2xla/graph_compiler.h @@ -69,23 +69,6 @@ class GraphCompiler { Status Compile(); private: - // NodeOutputs is a wrapper over TensorValues that represents outputs of a - // node. - struct NodeOutputs { - ~NodeOutputs() { - for (auto& v : values) { - CHECK(!v.is_ref()); - delete v.tensor; - } - } - - // Output values of this node. - std::vector values; - }; - - // A mapping from node id to node output. - using OutputRegistry = std::vector; - // Partially sets params. This partially set params can be reused // across multple nodes visit. void PartiallySetupParams(OpKernelContext::Params* params); -- GitLab From 7fe3744373751ee6a79bb23c6c20343a91d07b28 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Thu, 19 Oct 2017 14:01:53 -0700 Subject: [PATCH 162/573] Add local client execution test which uses infeed and outfeed. PiperOrigin-RevId: 172794367 --- .../xla/tests/local_client_execute_test.cc | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc index c74213f7f9..329b53012f 100644 --- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/compiler/xla/tests/test_utils.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/stream_executor_no_cuda.h" #include "tensorflow/core/platform/test.h" @@ -859,6 +860,31 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion64bit) { Literal::CreateR0(123456789000LL).get()})); } +// TODO(b/34359662): Support infeed/outfeed on GPU and CPU parallel. +// 2017-10-18. +XLA_TEST_F(LocalClientExecuteTest, + DISABLED_ON_GPU(DISABLED_ON_CPU_PARALLEL(InfeedOutfeedTest))) { + ComputationBuilder builder(local_client_, TestName()); + const Shape shape = ShapeUtil::MakeShape(F32, {3}); + auto in = builder.Infeed(shape); + auto constant = builder.ConstantR1({1.0f, 2.0f, 3.0f}); + auto sum = builder.Add(in, constant); + builder.Outfeed(sum, shape, /*outfeed_config=*/""); + + std::unique_ptr thread( + tensorflow::Env::Default()->StartThread( + tensorflow::ThreadOptions(), "execute_thread", + [&] { ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {}); })); + + ASSERT_IS_OK(local_client_->TransferToInfeed( + *Literal::CreateR1({-5.0, 123.0, 42.0}))); + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, + local_client_->TransferFromOutfeed(&shape)); + + LiteralTestUtil::ExpectR1Equal({-4.0, 125.0, 45.0}, *result); +} + // Benchmark that measures the overhead of the LocalClient API when running a // trivial computation void BM_LocalClientOverhead(int num_iters) { -- GitLab From eb978292e0ac46dd16c820b9989ad1776295517a Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 19 Oct 2017 14:16:16 -0700 Subject: [PATCH 163/573] Context-manager-based gradient API PiperOrigin-RevId: 172796719 --- tensorflow/contrib/eager/python/tfe.py | 2 + tensorflow/python/eager/backprop.py | 91 ++++++++++++++++++++++ tensorflow/python/eager/backprop_test.py | 21 +++++ tensorflow/python/eager/imperative_grad.py | 2 +- 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 353b9d2bda..25942aadfb 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -26,6 +26,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@implicit_value_and_gradients @@gradients_function @@value_and_gradients_function +@@GradientTape @@enable_tracing @@flush_trace @@ -92,5 +93,6 @@ implicit_gradients = backprop.implicit_grad implicit_value_and_gradients = backprop.implicit_val_and_grad gradients_function = backprop.gradients_function value_and_gradients_function = backprop.val_and_grad_function +GradientTape = backprop.GradientTape # pylint: disable=invalid-name remove_undocumented(__name__) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 61c905f31e..da17be05b7 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -36,6 +36,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.util import nest from tensorflow.python.util import tf_inspect @@ -696,3 +697,93 @@ _default_vspace = imperative_grad.VSpace( tensor_id=ops.tensor_id, zeros=array_ops.zeros, ones_like=array_ops.ones_like) + + +class GradientTape(object): + """Records operations to use to compute gradients. + + Operations are recorded if: + - they happen in code marked by this context manager + - at least one of their inputs is being watched + + Outputs of recorded operations are watched. Variables are automatically + watched and tensors can be manually watched by calling the watch method on the + context manager. + + Example usage: + + ```python + with tfe.GradientTape() as g: + x = tf.constant(3.0) + g.watch(x) + y = x * x + grad = g.gradient(y, [x])[0] + assert grad.numpy() == 6.0 + ``` + + It is possible to use GradientTapes to compute higher-order derivatives as + follows: + + ```python + with tfe.GradientTape() as g: + x = tf.constant(3.0) + g.watch(x) + y = x * x + with tfe.GradientTape() as gg: + gg.watch(y) + z = 2 * y + inner_grad = gg.gradient(z, [y])[0] + assert inner_grad.numpy() == 2 + y = y + inner_grad + grad = g.gradient(y, [x])[0] + assert grad.numpy() == 6.0 + ``` + """ + + def __init__(self): + self._tape = None + + def __enter__(self): + tape.push_new_tape() + return self + + def __exit__(self, typ, value, traceback): + self._tape = tape.pop_tape() + + def watch(self, tensor): + """Ensures that `tensor` is being traced by this tape. + + Args: + tensor: a Tensor or Variable a list of Tensors or Variables. + """ + for t in nest.flatten(tensor): + if isinstance(t, resource_variable_ops.ResourceVariable): + t = t.handle + tape.watch(t) + + def gradient(self, target, sources): + """Computes the gradient using information traced by the tape. + + Args: + target: the tensor to be differentiated. + sources: a list of Tensors or Variables, the target will be + differentiated with respect to the sources. + + Returns: + a list of Tensors (or IndexedSlices, or None), one for each element in + `sources`. + + Raises: + RuntimeError: if called inside the context of the tape, or if called more + than once. + """ + if self._tape is None: + raise RuntimeError("GradientTape.gradient can only be called once, and " + "only when the context manager has exited.") + sources = [x.handle if isinstance(x, resource_variable_ops.ResourceVariable) + else x + for x in sources] + grad = imperative_grad.imperative_grad( + _default_vspace, self._tape, [target], sources) + self.tape = None + return grad diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 5161095683..95d5f0adcb 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -277,6 +277,27 @@ class BackpropTest(test.TestCase): self.assertEqual(backprop.implicit_grad(f)()[0][0], None) + def testGradientTape(self): + with backprop.GradientTape() as g: + x = constant_op.constant(3.0) + g.watch(x) + y = x * x + with backprop.GradientTape() as gg: + gg.watch(y) + z = 2 * y + inner_grad = gg.gradient(z, [y])[0] + self.assertEqual(inner_grad.numpy(), 2.0) + y += inner_grad + grad = g.gradient(y, [x])[0] + self.assertEqual(grad.numpy(), 6.0) + + def testGradientTapeVariable(self): + v = resource_variable_ops.ResourceVariable(1.0) + with backprop.GradientTape() as g: + y = v * v + grad = g.gradient(y, [v])[0] + self.assertAllEqual(grad, 2.0) + def testEmptyParamsForValueAndGradFunction(self): def fn(a, b): return a * b diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py index ce58e661d7..c87719f84a 100644 --- a/tensorflow/python/eager/imperative_grad.py +++ b/tensorflow/python/eager/imperative_grad.py @@ -215,7 +215,7 @@ def imperative_grad( and tensor_usage_counts[t] == 0 and t not in id_sources): in_op = tensor_to_op[t] - if in_op is None: + if in_op is None or in_op == -1: continue if op_missing_tensor.get(in_op, 0) > 0: op_missing_tensor[in_op] -= 1 -- GitLab From ee7f8d973b9ea05495d799a728ab5ad9c654d125 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 14:22:29 -0700 Subject: [PATCH 164/573] Use "nullptr" instead of other null pointer constants PiperOrigin-RevId: 172797910 --- tensorflow/python/eager/pywrap_tfe_src.cc | 40 +++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 402b84d7c6..7456eb10f8 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -452,26 +452,26 @@ static void TFE_Py_Tape_Delete(PyObject* tape) { } static PyTypeObject TFE_Py_Tape_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "tfe.Tape", /* tp_name */ - sizeof(TFE_Py_Tape), /* tp_basicsize */ - 0, /* tp_itemsize */ - &TFE_Py_Tape_Delete, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "TFE_Py_Tape objects", /* tp_doc */ + PyVarObject_HEAD_INIT(nullptr, 0) "tfe.Tape", /* tp_name */ + sizeof(TFE_Py_Tape), /* tp_basicsize */ + 0, /* tp_itemsize */ + &TFE_Py_Tape_Delete, /* tp_dealloc */ + nullptr, /* tp_print */ + nullptr, /* tp_getattr */ + nullptr, /* tp_setattr */ + nullptr, /* tp_reserved */ + nullptr, /* tp_repr */ + nullptr, /* tp_as_number */ + nullptr, /* tp_as_sequence */ + nullptr, /* tp_as_mapping */ + nullptr, /* tp_hash */ + nullptr, /* tp_call */ + nullptr, /* tp_str */ + nullptr, /* tp_getattro */ + nullptr, /* tp_setattro */ + nullptr, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "TFE_Py_Tape objects", /* tp_doc */ }; PyObject* TFE_Py_NewTape() { -- GitLab From eeabcaffd502a5e9fb3664eaa89134c855a86148 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Oct 2017 14:23:03 -0700 Subject: [PATCH 165/573] `tf.py_func`: Handle NumPy arrays of np.object that hold unicode strings. This also fixes a bug affecting `tf.data.Dataset.from_generator()` on Python 3, where the generator yields Unicode (i.e. default) strings. PiperOrigin-RevId: 172798007 --- .../dataset_from_generator_op_test.py | 20 +++++++++++++++++++ .../python/kernel_tests/py_func_test.py | 15 ++++++++++++++ tensorflow/python/lib/core/py_func.cc | 11 ++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py index cd2bec8432..f129d07b57 100644 --- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py +++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py @@ -216,6 +216,26 @@ class DatasetConstructorTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + def testFromGeneratorString(self): + def generator(): + yield "foo" + yield b"bar" + yield u"baz" + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.string, output_shapes=[]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for expected in [b"foo", b"bar", b"baz"]: + next_val = sess.run(get_next) + self.assertAllEqual(expected, next_val) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + def testFromGeneratorTypeError(self): def generator(): yield np.array([1, 2, 3], dtype=np.int64) diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py index 4bd5b79797..7ed99c1be9 100644 --- a/tensorflow/python/kernel_tests/py_func_test.py +++ b/tensorflow/python/kernel_tests/py_func_test.py @@ -148,6 +148,21 @@ class PyOpTest(test.TestCase): z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"]) + def testObjectArraysAreConvertedToBytes(self): + + def read_object_array(): + return np.array([b" there", u" ya"], dtype=np.object) + + def read_and_return_strings(x, y): + return x + y + + with self.test_session(): + x = constant_op.constant(["hello", "hi"], dtypes.string) + y, = script_ops.py_func(read_object_array, [], + [dtypes.string]) + z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) + self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"]) + def testStringPadding(self): correct = [b"this", b"is", b"a", b"test"] with self.test_session(): diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc index 84cb4885f6..a62847614c 100644 --- a/tensorflow/python/lib/core/py_func.cc +++ b/tensorflow/python/lib/core/py_func.cc @@ -297,8 +297,15 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) { char* el; Py_ssize_t el_size; if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) { - return errors::Unimplemented("Unsupported object type ", - input_data[i]->ob_type->tp_name); +#if PY_MAJOR_VERSION >= 3 + el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size); + if (!el) { +#endif + return errors::Unimplemented("Unsupported object type ", + input_data[i]->ob_type->tp_name); +#if PY_MAJOR_VERSION >= 3 + } +#endif } tflat(i) = string(el, el_size); } -- GitLab From fe82a3165d1be801df64bd7dc3009ba8773ed4a9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 14:22:29 -0700 Subject: [PATCH 166/573] Use "nullptr" instead of other null pointer constants PiperOrigin-RevId: 172797910 --- .../dataset_from_generator_op_test.py | 20 ------------------- .../python/kernel_tests/py_func_test.py | 15 -------------- tensorflow/python/lib/core/py_func.cc | 11 ++-------- 3 files changed, 2 insertions(+), 44 deletions(-) diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py index f129d07b57..cd2bec8432 100644 --- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py +++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py @@ -216,26 +216,6 @@ class DatasetConstructorTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) - def testFromGeneratorString(self): - def generator(): - yield "foo" - yield b"bar" - yield u"baz" - - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.string, output_shapes=[]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - for expected in [b"foo", b"bar", b"baz"]: - next_val = sess.run(get_next) - self.assertAllEqual(expected, next_val) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - def testFromGeneratorTypeError(self): def generator(): yield np.array([1, 2, 3], dtype=np.int64) diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py index 7ed99c1be9..4bd5b79797 100644 --- a/tensorflow/python/kernel_tests/py_func_test.py +++ b/tensorflow/python/kernel_tests/py_func_test.py @@ -148,21 +148,6 @@ class PyOpTest(test.TestCase): z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"]) - def testObjectArraysAreConvertedToBytes(self): - - def read_object_array(): - return np.array([b" there", u" ya"], dtype=np.object) - - def read_and_return_strings(x, y): - return x + y - - with self.test_session(): - x = constant_op.constant(["hello", "hi"], dtypes.string) - y, = script_ops.py_func(read_object_array, [], - [dtypes.string]) - z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) - self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"]) - def testStringPadding(self): correct = [b"this", b"is", b"a", b"test"] with self.test_session(): diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc index a62847614c..84cb4885f6 100644 --- a/tensorflow/python/lib/core/py_func.cc +++ b/tensorflow/python/lib/core/py_func.cc @@ -297,15 +297,8 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) { char* el; Py_ssize_t el_size; if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) { -#if PY_MAJOR_VERSION >= 3 - el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size); - if (!el) { -#endif - return errors::Unimplemented("Unsupported object type ", - input_data[i]->ob_type->tp_name); -#if PY_MAJOR_VERSION >= 3 - } -#endif + return errors::Unimplemented("Unsupported object type ", + input_data[i]->ob_type->tp_name); } tflat(i) = string(el, el_size); } -- GitLab From 17ba3a69f4c3509711a3da5eff3cb6be99e0936d Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Oct 2017 14:23:03 -0700 Subject: [PATCH 167/573] `tf.py_func`: Handle NumPy arrays of np.object that hold unicode strings. This also fixes a bug affecting `tf.data.Dataset.from_generator()` on Python 3, where the generator yields Unicode (i.e. default) strings. PiperOrigin-RevId: 172798007 --- .../dataset_from_generator_op_test.py | 20 +++++++++++++++++++ .../python/kernel_tests/py_func_test.py | 15 ++++++++++++++ tensorflow/python/lib/core/py_func.cc | 11 ++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py index cd2bec8432..f129d07b57 100644 --- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py +++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py @@ -216,6 +216,26 @@ class DatasetConstructorTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + def testFromGeneratorString(self): + def generator(): + yield "foo" + yield b"bar" + yield u"baz" + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.string, output_shapes=[]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for expected in [b"foo", b"bar", b"baz"]: + next_val = sess.run(get_next) + self.assertAllEqual(expected, next_val) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + def testFromGeneratorTypeError(self): def generator(): yield np.array([1, 2, 3], dtype=np.int64) diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py index 4bd5b79797..7ed99c1be9 100644 --- a/tensorflow/python/kernel_tests/py_func_test.py +++ b/tensorflow/python/kernel_tests/py_func_test.py @@ -148,6 +148,21 @@ class PyOpTest(test.TestCase): z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"]) + def testObjectArraysAreConvertedToBytes(self): + + def read_object_array(): + return np.array([b" there", u" ya"], dtype=np.object) + + def read_and_return_strings(x, y): + return x + y + + with self.test_session(): + x = constant_op.constant(["hello", "hi"], dtypes.string) + y, = script_ops.py_func(read_object_array, [], + [dtypes.string]) + z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) + self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"]) + def testStringPadding(self): correct = [b"this", b"is", b"a", b"test"] with self.test_session(): diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc index 84cb4885f6..a62847614c 100644 --- a/tensorflow/python/lib/core/py_func.cc +++ b/tensorflow/python/lib/core/py_func.cc @@ -297,8 +297,15 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) { char* el; Py_ssize_t el_size; if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) { - return errors::Unimplemented("Unsupported object type ", - input_data[i]->ob_type->tp_name); +#if PY_MAJOR_VERSION >= 3 + el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size); + if (!el) { +#endif + return errors::Unimplemented("Unsupported object type ", + input_data[i]->ob_type->tp_name); +#if PY_MAJOR_VERSION >= 3 + } +#endif } tflat(i) = string(el, el_size); } -- GitLab From 21d2de1c8d34d5094472dd828394c239d6111e0d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 15:02:01 -0700 Subject: [PATCH 168/573] Add a recursive descent parser for the HloModule string. It constructs an HloModule object from a string printed by HloModule::ToString(). This is a initial stage. It currently supports: - unary, binary, ternary ops, and other ops that don't have extra attributes. - module with entry computation only. - simple cases for constant instruction. To make the parser simpler, this cl removes a whitespace and adds a '%' before the computation name in HloComputation::ToString(). Further steps will enable parsing subcomputations, more cases of constants, tuple, and ops that require extra attributes (e.g., broadcast dimensions, subcomputation). PiperOrigin-RevId: 172804214 --- tensorflow/BUILD | 1 + .../compiler/xla/service/hlo_computation.cc | 4 +- tensorflow/compiler/xla/shape_util.cc | 45 +- tensorflow/compiler/xla/tools/parser/BUILD | 84 +++ .../compiler/xla/tools/parser/README.md | 69 +++ .../compiler/xla/tools/parser/hlo_lexer.cc | 270 ++++++++++ .../compiler/xla/tools/parser/hlo_lexer.h | 108 ++++ .../compiler/xla/tools/parser/hlo_parser.cc | 502 ++++++++++++++++++ .../compiler/xla/tools/parser/hlo_parser.h | 37 ++ .../xla/tools/parser/hlo_parser_test.cc | 240 +++++++++ .../compiler/xla/tools/parser/hlo_token.h | 58 ++ 11 files changed, 1402 insertions(+), 16 deletions(-) create mode 100644 tensorflow/compiler/xla/tools/parser/BUILD create mode 100644 tensorflow/compiler/xla/tools/parser/README.md create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.cc create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.h create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.cc create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.h create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_token.h diff --git a/tensorflow/BUILD b/tensorflow/BUILD index e351037abb..d5c56cdc18 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -340,6 +340,7 @@ filegroup( "//tensorflow/compiler/xla/service/llvm_ir:all_files", "//tensorflow/compiler/xla/tests:all_files", "//tensorflow/compiler/xla/tools:all_files", + "//tensorflow/compiler/xla/tools/parser:all_files", "//tensorflow/contrib:all_files", "//tensorflow/contrib/all_reduce:all_files", "//tensorflow/contrib/android:all_files", diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 9b3104eaac..51ead753f0 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -373,8 +373,8 @@ string HloComputation::ToString(int nested_level) const { for (int i = 0; i < nested_level; i++) { s << " "; } - s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) - << " { \n"; + s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) + << " {\n"; for (const HloInstruction* instruction : MakeInstructionPostOrder()) { for (int i = 0; i < nested_level; i++) { s << " "; diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index 8e16056b23..af583bed62 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -102,6 +102,32 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { return true; } +// Constructs and returns the new shape with the given minor_to_major order in +// its Layout. +StatusOr MakeShapeWithLayoutInternal( + PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, + tensorflow::gtl::ArraySlice minor_to_major) { + if (dimensions.size() != minor_to_major.size()) { + return InvalidArgument("Dimensions size is %ld, but layout size is %ld.", + dimensions.size(), minor_to_major.size()); + } + if (element_type == OPAQUE || element_type == TUPLE) { + return InvalidArgument("Unsupported element type: %s", + PrimitiveType_Name(element_type).c_str()); + } + Shape shape = ShapeUtil::MakeShape(element_type, dimensions); + auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); + min2maj->Clear(); + for (int64 value : minor_to_major) { + min2maj->Add(value); + } + if (!shape.has_layout()) { + return InvalidArgument("Shape has no layout."); + } + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape)); + return shape; +} + } // namespace /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) { @@ -152,16 +178,8 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { /* static */ Shape ShapeUtil::MakeShapeWithLayout( PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, tensorflow::gtl::ArraySlice minor_to_major) { - CHECK_EQ(dimensions.size(), minor_to_major.size()); - Shape shape = MakeShape(element_type, dimensions); - auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); - min2maj->Clear(); - for (int64 value : minor_to_major) { - min2maj->Add(value); - } - DCHECK(shape.has_layout()); - TF_DCHECK_OK(ValidateShape(shape)); - return shape; + return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major) + .ValueOrDie(); } /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout( @@ -499,11 +517,10 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { // Extract the layout minor-to-major and set it. TF_ASSIGN_OR_RETURN(std::vector min2maj, comma_list_to_int64s(layout_string)); - TF_RET_CHECK(dimensions.size() == min2maj.size()); - result = - ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj); + TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal( + primitive_type, dimensions, min2maj)); } - TF_DCHECK_OK(ShapeUtil::ValidateShape(result)); + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result)); return std::move(result); } diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD new file mode 100644 index 0000000000..c84ca9fc83 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/BUILD @@ -0,0 +1,84 @@ +# Build file for the Hlo parser. + +licenses(["notice"]) # Apache 2.0 + +package( + default_visibility = [":friends"], +) + +package_group( + name = "friends", + includes = [ + "//tensorflow/compiler/xla:friends", + ], +) + +# Filegroup used to collect source files for dependency checking. +filegroup( + name = "c_srcs", + data = glob([ + "**/*.cc", + "**/*.h", + ]), +) + +load("//tensorflow:tensorflow.bzl", "tf_cc_test") + +cc_library( + name = "hlo_lexer", + srcs = ["hlo_lexer.cc"], + hdrs = [ + "hlo_lexer.h", + "hlo_token.h", + ], + deps = [ + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core:lib", + "//tensorflow/core:regexp_internal", + ], +) + +cc_library( + name = "hlo_parser", + srcs = ["hlo_parser.cc"], + hdrs = ["hlo_parser.h"], + deps = [ + ":hlo_lexer", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + ], +) + +tf_cc_test( + name = "hlo_parser_test", + size = "small", + srcs = ["hlo_parser_test.cc"], + deps = [ + ":hlo_parser", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + +# ----------------------------------------------------------------------------- + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md new file mode 100644 index 0000000000..a334bc2b29 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/README.md @@ -0,0 +1,69 @@ +# HloModule string syntax + +TODO: Support subcomputations (for fusion, reduce, while, ...). + +TODO: Support ops that require extra attributes, e.g. dimensions, strides. + +```yacc +hlo_module + : 'HloModule' name computation + ; + +computation + : 'ENTRY' name param_list '->' shape instruction_list + ; + +instruction_list + : '{' instruction_list1 '}' + ; +instruction_list1 + : instruction + | instruction_list1 instruction + ; +instruction + : name '=' shape opcode operands + ; + +operands + : '(' operands1 ')' + ; +operands1 + : /*empty*/ + | operand + | operands1 ',' operand + ; +operand + : shape name + ; + +param_list + : '(' param_list1 ')' + ; +param_list1 + : /*empty*/ + | param + | param_list1 ',' param + ; +param + : name shape + ; + +shape + : shape_val_ + | '(' tuple_elements ')' + ; +tuple_elements + : /*empty*/ + | shape (',' shape)* + ; + +name + : identifier ':' + | '%' identifier + ; + +identifier + : [a-zA-Z_][a-zA-Z0-9_.-]* + ; + +``` diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc new file mode 100644 index 0000000000..3e84ffcbd2 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc @@ -0,0 +1,270 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" + +#include + +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/gtl/optional.h" +#include "tensorflow/core/lib/strings/numbers.h" +#include "tensorflow/core/platform/regexp.h" + +namespace xla { +namespace tools { + +using tensorflow::StringPiece; + +namespace { + +constexpr int kEOF = -1; +constexpr int kError = -2; + +// [a-zA-Z0-9_.-] +bool IsIdentifierChar(char c) { + return isalnum(static_cast(c)) || c == '-' || c == '.' || + c == '_'; +} + +} // namespace + +int HloLexer::GetNextChar() { + int current_char = PeekCurrentChar(); + if (current_char != kEOF && current_char != kError) { + current_ptr_++; + } + return current_char; +} + +int HloLexer::PeekCurrentChar() const { + if (current_ptr_ == buf_.end()) { + return kEOF; + } + char current_char = *current_ptr_; + if (current_char == 0) { + // '\0' should not appear in the middle of the string. + return kError; + } + return static_cast(current_char); +} + +bool HloLexer::CanDereference(const char* ptr) const { + return ptr < buf_.end() && ptr >= buf_.begin(); +} + +StringPiece HloLexer::StringPieceFromPointers(const char* begin, + const char* end) const { + CHECK(begin <= end); + CHECK(begin == buf_.end() || CanDereference(begin)); + CHECK(end == buf_.end() || CanDereference(end)); + return StringPiece(begin, end - begin); +} + +tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers( + const char* begin, const char* end) const { + CHECK(begin <= end); + CHECK(begin == buf_.end() || CanDereference(begin)); + CHECK(end == buf_.end() || CanDereference(end)); + return tensorflow::RegexpStringPiece(begin, end - begin); +} + +TokKind HloLexer::LexToken() { + while (true) { + token_start_ = current_ptr_; + + int current_char = GetNextChar(); + switch (current_char) { + default: + // [a-zA-Z_] + if (isalpha(static_cast(current_char)) || + current_char == '_') { + return LexIdentifier(); + } + return TokKind::kError; + case kEOF: + // Hit the end of the input buffer. + return TokKind::kEof; + case kError: + // Hit an invalid character in the input buffer. + return TokKind::kError; + case ' ': + case '\t': + case '\n': + case '\r': + // Ignore whitespace. + continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (current_char == '-' && PeekCurrentChar() == '>') { + current_ptr_++; + return TokKind::kArrow; + } + return LexDigitOrNegative(); + case '=': + return TokKind::kEqual; + case ',': + return TokKind::kComma; + case '%': + return LexPercent(); + case ':': + return TokKind::kColon; + case '[': + return TokKind::kLsquare; + case ']': + return TokKind::kRsquare; + case '{': + return TokKind::kLbrace; + case '}': + return TokKind::kRbrace; + case '(': + return TokKind::kLparen; + case ')': + return TokKind::kRparen; + } + } +} + +// Lex a shape, name, keyword, or opcode. +// shape ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})? +// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*: +// keyword ::= HloModule, ENTRY, ... +// opcode ::= add, greater-than, ... +TokKind HloLexer::LexIdentifier() { + { + auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); + // 'consumable' will be advanced iff its prefix matches the pattern. + static LazyRE2 shape_pattern = { + R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"}; + if (RE2::Consume(&consumable, *shape_pattern)) { + auto status_or_shape = ShapeUtil::ParseShapeString( + StringPieceFromPointers(token_start_, consumable.begin())); + if (status_or_shape.ok()) { + // This is a shape string. + shape_val_ = status_or_shape.ValueOrDie(); + current_ptr_ = consumable.begin(); + return TokKind::kShape; + } + } + } + + while (IsIdentifierChar(PeekCurrentChar())) { + current_ptr_++; + } + + // If followed by ':', it's a name. + if (PeekCurrentChar() == ':') { + str_val_.assign(token_start_, current_ptr_); + current_ptr_++; // skip ':' + return TokKind::kName; + } + + StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_); + + // See if this is a keyword. +#define KEYWORD(STR) \ + do { \ + if (identifier == #STR) { \ + return TokKind::kw_##STR; \ + } \ + } while (false) + + KEYWORD(true); + KEYWORD(false); + KEYWORD(HloModule); + KEYWORD(ENTRY); + +#undef KEYWORD + + // See if this is an opcode. + auto opcode = StringToHloOpcode(identifier.ToString()); + if (opcode.ok()) { + opcode_val_ = opcode.ValueOrDie(); + return TokKind::kOpcode; + } + + current_ptr_ = token_start_ + 1; + return TokKind::kError; +} + +// Lex names after a % character. +// name ::= [a-zA-Z_][a-zA-Z0-9_.-]* +TokKind HloLexer::LexPercent() { + const char* name_start = current_ptr_; + if (isalpha(static_cast(PeekCurrentChar())) || + PeekCurrentChar() == '_') { + current_ptr_++; + while (IsIdentifierChar(PeekCurrentChar())) { + current_ptr_++; + } + str_val_.assign(name_start, current_ptr_); + return TokKind::kName; + } + return TokKind::kError; +} + +// Lex integer and floating-point values. +// int [-]?[0-9]+ +// fp with exp [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+) +// fp without exp [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+) +TokKind HloLexer::LexDigitOrNegative() { + auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); + static LazyRE2 float_pattern = { + R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"}; + if (RE2::Consume(&consumable, *float_pattern)) { + current_ptr_ = consumable.begin(); + tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(), + &decimal_val_); + return TokKind::kDecimal; + } + + static LazyRE2 int_pattern = {R"([-]?\d+)"}; + if (RE2::Consume(&consumable, *int_pattern)) { + current_ptr_ = consumable.begin(); + tensorflow::strings::safe_strto64( + StringPieceFromPointers(token_start_, current_ptr_), &int64_val_); + return TokKind::kInt; + } + + return TokKind::kError; +} + +StringPiece HloLexer::GetCurrentLine() const { + const char* start = token_start_; + const char* end = current_ptr_; + if (!CanDereference(start) || !CanDereference(end)) { + return "LINE OUT OF RANGE"; + } + while (start > buf_.begin() && *start != '\n') { + start--; + } + while (end < buf_.end() && *end != '\n') { + end++; + } + return StringPieceFromPointers(start, end); +} + +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h new file mode 100644 index 0000000000..20278fd6cd --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h @@ -0,0 +1,108 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ + +#include + +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_token.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/regexp.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { +namespace tools { + +// Lexer for the HloModule::ToString() format text. +class HloLexer { + public: + explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) { + current_ptr_ = buf_.begin(); + } + + TokKind Lex() { return current_kind_ = LexToken(); } + TokKind GetKind() const { return current_kind_; } + string GetStrVal() const { + CHECK(GetKind() == TokKind::kName); + return str_val_; + } + Shape GetShapeVal() const { + CHECK(GetKind() == TokKind::kShape); + return shape_val_; + } + HloOpcode GetOpcodeVal() const { + CHECK(GetKind() == TokKind::kOpcode); + return opcode_val_; + } + int64 GetInt64Val() const { + CHECK(GetKind() == TokKind::kInt); + return int64_val_; + } + double GetDecimalVal() const { + CHECK(GetKind() == TokKind::kDecimal); + return decimal_val_; + } + + // Returns the line of text that is currently being lexed. + tensorflow::StringPiece GetCurrentLine() const; + + private: + // Returns the current character. If it's neither the end of input buffer nor + // an invalid character, moves the pointer forward. + int GetNextChar(); + + // Returns the current character. + int PeekCurrentChar() const; + + // Creates StringPiece with the given begin and end. Exits if the begin > end, + // or it's out of the range of the current buffer. + tensorflow::StringPiece StringPieceFromPointers(const char* begin, + const char* end) const; + tensorflow::RegexpStringPiece RegexpStringPieceFromPointers( + const char* begin, const char* end) const; + + // Returns true if the given ptr is dereferenceable within the range of the + // current buffer. + bool CanDereference(const char* ptr) const; + + TokKind LexToken(); + + TokKind LexIdentifier(); + TokKind LexPercent(); + TokKind LexShape(); + TokKind LexConstant(); + TokKind LexDigitOrNegative(); + + const tensorflow::StringPiece buf_; + const char* current_ptr_; + + // Information about the current token. + const char* token_start_; + TokKind current_kind_; + string str_val_; + Shape shape_val_; + HloOpcode opcode_val_; + int64 int64_val_; + double decimal_val_; +}; + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc new file mode 100644 index 0000000000..57700493e6 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -0,0 +1,502 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" + +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/core/lib/gtl/map_util.h" +#include "tensorflow/core/lib/strings/strcat.h" + +namespace xla { +namespace tools { + +namespace { + +using tensorflow::StringPiece; +using tensorflow::strings::StrCat; + +// Parser for the HloModule::ToString() format text. +class HloParser { + public: + explicit HloParser(StringPiece str) : lexer_(str) {} + + // Runs the parser. Returns false if an error occurred. + bool Run(); + + // Returns the parsed HloModule. + std::unique_ptr ConsumeHloModule() { return std::move(module_); } + + // Returns the error information. + string GetError() const { return tensorflow::str_util::Join(error_, "\n"); } + + private: + // ParseXXX returns false if an error occurred. + bool ParseHloModule(); + bool ParseComputation(); + bool ParseInstructionList(HloComputation::Builder* builder); + bool ParseInstruction(HloComputation::Builder* builder); + bool ParseLiteral(std::unique_ptr* literal, const Shape& shape); + bool ParseOperands(std::vector* operands, + const int expected_size); + bool ParseParamList(); + bool ParseName(string* result); + bool ParseShape(Shape* result); + bool ParseOpcode(HloOpcode* result); + bool ParseInt64(int64* result); + bool ParseDecimal(double* result); + bool ParseBool(bool* result); + bool ParseToken(TokKind kind, const string& msg); + + // Logs the current parsing line and the given message. Always returns false. + bool TokenError(StringPiece msg); + + // If the current token is 'kind', eats it (i.e. lexes the next token) and + // returns true. + bool EatIfPresent(TokKind kind); + + // Adds the instruction to the pool. Returns false and emits an error if the + // instruction already exists. + bool AddInstruction(const string& name, HloInstruction* instruction); + + // The map from the instruction name to the instruction. This does not own the + // instructions. + std::unordered_map instruction_pool_; + + HloLexer lexer_; + std::unique_ptr module_; + std::vector error_; +}; + +bool HloParser::TokenError(StringPiece msg) { + error_.push_back( + StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg)); + return false; +} + +bool HloParser::Run() { + lexer_.Lex(); + return ParseHloModule(); +} + +// ::= 'HloModule' name computation +bool HloParser::ParseHloModule() { + if (lexer_.GetKind() != TokKind::kw_HloModule) { + return TokenError("expects HloModule"); + } + // Eat 'HloModule' + lexer_.Lex(); + + string name; + if (!ParseName(&name)) { + return false; + } + + module_ = MakeUnique(name); + + return ParseComputation(); +} + +// computation ::= 'ENTRY' name param_list '->' shape instruction_list +bool HloParser::ParseComputation() { + string name; + if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) { + return false; + } + auto builder = MakeUnique(name); + + Shape shape; + if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") || + !ParseShape(&shape) || !ParseInstructionList(builder.get())) { + return false; + } + module_->AddEntryComputation(builder->Build()); + return true; +} + +// instruction_list ::= '{' instruction_list1 '}' +// instruction_list1 ::= (instruction)+ +bool HloParser::ParseInstructionList(HloComputation::Builder* builder) { + if (!ParseToken(TokKind::kLbrace, + "expects '{' at the beginning of instruction list.")) { + return false; + } + do { + if (!ParseInstruction(builder)) { + return false; + } + } while (lexer_.GetKind() != TokKind::kRbrace); + return ParseToken(TokKind::kRbrace, + "expects '}' at the end of instruction list."); +} + +// instruction ::= name '=' shape opcode operands +bool HloParser::ParseInstruction(HloComputation::Builder* builder) { + string name; + Shape shape; + HloOpcode opcode; + std::vector operands; + if (!ParseName(&name) || + !ParseToken(TokKind::kEqual, "expects '=' in instruction") || + !ParseShape(&shape) || !ParseOpcode(&opcode)) { + return false; + } + switch (opcode) { + case HloOpcode::kParameter: { + int64 parameter_number; + return ParseToken(TokKind::kLparen, + "expects '(' before parameter number") && + ParseInt64(¶meter_number) && + ParseToken(TokKind::kRparen, + "expects ')' after parameter number") && + AddInstruction( + name, builder->AddInstruction(HloInstruction::CreateParameter( + parameter_number, shape, name))); + } + case HloOpcode::kConstant: { + std::unique_ptr literal; + return ParseToken(TokKind::kLparen, + "expects '(' before parameter number") && + ParseLiteral(&literal, shape) && + ParseToken(TokKind::kRparen, + "expects ')' after parameter number") && + AddInstruction( + name, builder->AddInstruction( + HloInstruction::CreateConstant(std::move(literal)))); + } + // Unary ops. + case HloOpcode::kAbs: + case HloOpcode::kRoundNearestAfz: + case HloOpcode::kBitcast: + case HloOpcode::kCeil: + case HloOpcode::kCopy: + case HloOpcode::kCos: + case HloOpcode::kExp: + case HloOpcode::kIsFinite: + case HloOpcode::kFloor: + case HloOpcode::kLog: + case HloOpcode::kNot: + case HloOpcode::kNegate: + case HloOpcode::kSign: + case HloOpcode::kSin: + case HloOpcode::kSort: + case HloOpcode::kTanh: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction(name, + builder->AddInstruction(HloInstruction::CreateUnary( + shape, opcode, operands[0]))); + } + // Binary ops. + case HloOpcode::kAdd: + case HloOpcode::kDivide: + case HloOpcode::kMultiply: + case HloOpcode::kSubtract: + case HloOpcode::kEq: + case HloOpcode::kGe: + case HloOpcode::kGt: + case HloOpcode::kLe: + case HloOpcode::kLt: + case HloOpcode::kNe: + case HloOpcode::kDot: + case HloOpcode::kMaximum: + case HloOpcode::kMinimum: + case HloOpcode::kPower: + case HloOpcode::kRemainder: + case HloOpcode::kAnd: + case HloOpcode::kOr: + case HloOpcode::kShiftLeft: + case HloOpcode::kShiftRightArithmetic: + case HloOpcode::kShiftRightLogical: { + return ParseOperands(&operands, /*expected_size=*/2) && + AddInstruction( + name, builder->AddInstruction(HloInstruction::CreateBinary( + shape, opcode, operands[0], operands[1]))); + } + // Ternary ops. + case HloOpcode::kClamp: + case HloOpcode::kSelect: { + return ParseOperands(&operands, /*expected_size=*/3) && + AddInstruction( + name, + builder->AddInstruction(HloInstruction::CreateTernary( + shape, opcode, operands[0], operands[1], operands[2]))); + } + // Other supported ops. + case HloOpcode::kConvert: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction( + name, builder->AddInstruction( + HloInstruction::CreateConvert(shape, operands[0]))); + } + case HloOpcode::kCrossReplicaSum: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction(name, builder->AddInstruction( + HloInstruction::CreateCrossReplicaSum( + shape, operands[0]))); + } + case HloOpcode::kReshape: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction( + name, builder->AddInstruction( + HloInstruction::CreateReshape(shape, operands[0]))); + } + case HloOpcode::kBroadcast: + case HloOpcode::kCall: + case HloOpcode::kCustomCall: + case HloOpcode::kConcatenate: + case HloOpcode::kReducePrecision: + case HloOpcode::kConvolution: + case HloOpcode::kGetTupleElement: + case HloOpcode::kMap: + case HloOpcode::kPad: + case HloOpcode::kReduce: + case HloOpcode::kReduceWindow: + case HloOpcode::kSelectAndScatter: + case HloOpcode::kReverse: + case HloOpcode::kRng: + case HloOpcode::kSlice: + case HloOpcode::kDynamicSlice: + case HloOpcode::kDynamicUpdateSlice: + case HloOpcode::kTranspose: + case HloOpcode::kTuple: + case HloOpcode::kWhile: + case HloOpcode::kFusion: + case HloOpcode::kBatchNormTraining: + case HloOpcode::kBatchNormInference: + case HloOpcode::kInfeed: + case HloOpcode::kOutfeed: + case HloOpcode::kBatchNormGrad: + case HloOpcode::kRecv: + case HloOpcode::kSend: + case HloOpcode::kUpdate: + case HloOpcode::kIndex: + case HloOpcode::kTrace: + return TokenError(StrCat("parsing not yet implemented for op: ", + HloOpcodeString(opcode))); + } +} + +bool HloParser::ParseLiteral(std::unique_ptr* literal, + const Shape& shape) { + switch (shape.element_type()) { + case PRED: + bool b; + if (!ParseBool(&b)) { + return false; + } + *literal = Literal::CreateR0(b); + return true; + case S32: + int64 i; + if (!ParseInt64(&i)) { + return false; + } + *literal = Literal::CreateR0(i); + return true; + case F32: + double d; + if (!ParseDecimal(&d)) { + return false; + } + *literal = Literal::CreateR0(d); + return true; + default: + return TokenError(StrCat("unsupported constant in shape: ", + ShapeUtil::HumanString(shape))); + } +} + +// operands ::= '(' operands1 ')' +// operands1 +// ::= /*empty*/ +// ::= operand (, operand)* +// operand ::= shape name +bool HloParser::ParseOperands(std::vector* operands, + const int expected_size) { + if (!ParseToken(TokKind::kLparen, + "expects '(' at the beginning of operands")) { + return false; + } + if (lexer_.GetKind() == TokKind::kRparen) { + // empty + } else { + do { + Shape shape; + string name; + if (!ParseShape(&shape) || !ParseName(&name)) { + return false; + } + HloInstruction* instruction = + tensorflow::gtl::FindPtrOrNull(instruction_pool_, name); + if (!instruction) { + return TokenError(StrCat("instruction does not exist: ", name)); + } + operands->push_back(instruction); + } while (EatIfPresent(TokKind::kComma)); + } + if (expected_size != operands->size()) { + return TokenError(StrCat("expects ", expected_size, " operands, but has ", + operands->size(), " operands")); + } + return ParseToken(TokKind::kRparen, "expects ')' at the end of operands"); +} + +// param_list ::= '(' param_list1 ')' +// param_list1 +// ::= /*empty*/ +// ::= param (',' param)* +// param ::= name shape +bool HloParser::ParseParamList() { + if (!ParseToken(TokKind::kLparen, + "expects '(' at the beginning of param list")) { + return false; + } + + if (lexer_.GetKind() == TokKind::kRparen) { + // empty + } else { + do { + Shape shape; + if (!ParseToken(TokKind::kName, "expects name in parameter") || + !ParseShape(&shape)) { + return false; + } + } while (EatIfPresent(TokKind::kComma)); + } + return ParseToken(TokKind::kRparen, "expects ')' at the end of param list"); +} + +// shape ::= shape_val_ +// shape ::= '(' tuple_elements ')' +// tuple_elements +// ::= /*empty*/ +// ::= shape (',' shape)* +bool HloParser::ParseShape(Shape* result) { + if (EatIfPresent(TokKind::kLparen)) { // Tuple + std::vector shapes; + if (lexer_.GetKind() == TokKind::kRparen) { + /*empty*/ + } else { + // shape (',' shape)* + do { + shapes.emplace_back(); + if (!ParseShape(&shapes.back())) { + return false; + } + } while (EatIfPresent(TokKind::kComma)); + } + *result = ShapeUtil::MakeTupleShape(shapes); + return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple."); + } + + if (lexer_.GetKind() != TokKind::kShape) { + return TokenError("expects shape"); + } + *result = lexer_.GetShapeVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseName(string* result) { + VLOG(1) << "ParseName"; + if (lexer_.GetKind() != TokKind::kName) { + return TokenError("expects name"); + } + *result = lexer_.GetStrVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseOpcode(HloOpcode* result) { + VLOG(1) << "ParseOpcode"; + if (lexer_.GetKind() != TokKind::kOpcode) { + return TokenError("expects opcode"); + } + *result = lexer_.GetOpcodeVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseInt64(int64* result) { + VLOG(1) << "ParseInt64"; + if (lexer_.GetKind() != TokKind::kInt) { + return TokenError("expects integer"); + } + *result = lexer_.GetInt64Val(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseDecimal(double* result) { + switch (lexer_.GetKind()) { + case TokKind::kDecimal: + *result = lexer_.GetDecimalVal(); + break; + case TokKind::kInt: + *result = static_cast(lexer_.GetInt64Val()); + break; + default: + return TokenError("expects decimal or integer"); + } + lexer_.Lex(); + return true; +} + +bool HloParser::ParseBool(bool* result) { + if (lexer_.GetKind() != TokKind::kw_true && + lexer_.GetKind() != TokKind::kw_false) { + return TokenError("expects true or false"); + } + *result = lexer_.GetKind() == TokKind::kw_true; + lexer_.Lex(); + return true; +} + +bool HloParser::ParseToken(TokKind kind, const string& msg) { + if (lexer_.GetKind() != kind) { + return TokenError(msg); + } + lexer_.Lex(); + return true; +} + +bool HloParser::EatIfPresent(TokKind kind) { + if (lexer_.GetKind() != kind) { + return false; + } + lexer_.Lex(); + return true; +} + +bool HloParser::AddInstruction(const string& name, + HloInstruction* instruction) { + auto result = instruction_pool_.insert({name, instruction}); + if (!result.second) { + return TokenError(StrCat("instruction already exists: ", name)); + } + return true; +} + +} // namespace + +StatusOr> Parse(StringPiece str) { + HloParser parser(str); + if (!parser.Run()) { + return InvalidArgument("Syntax error: %s", parser.GetError().c_str()); + } + return parser.ConsumeHloModule(); +} + +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h new file mode 100644 index 0000000000..9aaf18ef20 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.h @@ -0,0 +1,37 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ + +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" + +namespace xla { +namespace tools { + +// The api of the hlo parser. Given a string in the HloModule::ToString() +// format, returns the parsed HloModule. +StatusOr> Parse(tensorflow::StringPiece str); + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc new file mode 100644 index 0000000000..4ecece3eac --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -0,0 +1,240 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" + +#include +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { +namespace tools { +namespace { + +struct TestData { + string test_name; + string module_string; +}; + +string TestDataToString(const ::testing::TestParamInfo& data) { + return data.param.test_name; +} + +std::vector CreateTestCases() { + // clang-format off + return std::vector({ +// ax + y +{ +"AxpyParam", +R"(HloModule axpy_module: + +ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { + %alpha = f32[2,4]{1,0} parameter(0) + %x = f32[2,4]{1,0} parameter(1) + %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x) + %y = f32[2,4]{1,0} parameter(2) + %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y) +} + +)" +}, +// pred constant +{ +"ConstantPred", +R"(HloModule constant_pred_module: + +ENTRY %constant_pred () -> pred[] { + %constant = pred[] constant(true) +} + +)" +}, +// s32 constant +{ +"ConstantS32", +R"(HloModule constant_s32_module: + +ENTRY %constant_s32 () -> s32[] { + %constant = s32[] constant(-42) +} + +)" +}, +// f32 constant, but the value is not a decimal +{ +"ConstantF32", R"(HloModule ConstantF32_module: + +ENTRY %ConstantF32.v4 () -> f32[] { + %constant = f32[] constant(42) +} + +)" +}, +// constant + constant +{ +"AddConstants", +R"(HloModule add_constants_module: + +ENTRY %add_constants () -> f32[] { + %constant = f32[] constant(3.14) + %add = f32[] add(f32[] %constant, f32[] %constant) +} + +)" +}, +// v1 > v2 ? v1 : v2 +{ +"SelectR1F32", +R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module: + +ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { + %v1 = f32[4]{0} parameter(0) + %v2 = f32[4]{0} parameter(1) + %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2) + %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) +} + +)" +} + }); + // clang-format on +} + +class HloParserTest : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + void ExpectSuccess() { + const string& original = GetParam().module_string; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + EXPECT_EQ(original, result.ValueOrDie()->ToString()); + } +}; + +TEST_P(HloParserTest, Run) { ExpectSuccess(); } + +INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest, + ::testing::ValuesIn(CreateTestCases()), + TestDataToString); + +TEST_F(HloParserTest, Empty) { + const string original = ""; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, Garbage) { + const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongOpcode) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: f32[], y: f32[]) -> f32[] { + %x = f32[]{} parameter(0) + %y = f32[]{} parameter(1) + %le = pred[]{} le(f32[]{} %x, f32[]{} %y) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongShape) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: g32[]) -> g32[] { + %x = g32[]{} parameter(0) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongOperandsSize) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: f32[]) -> pred[] { + %x = f32[]{} parameter(0) + %eq = pred[]{} equal-to(f32[]{} %x) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, OperandNotFound) { + const string original = R"(HloModule operand_not_found: +ENTRY %blabla (x: f32[]) -> pred[] { + %x = f32[]{} parameter(0) + %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y) +} +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, MoreConstants) { + const string original = R"(HloModule SelectScalarS32True_module: + +ENTRY %SelectScalarS32True.v4 () -> s32[] { + %constant.2 = pred[] constant(true) + %constant.1 = s32[] constant(-42) + %constant = s32[] constant(42) + %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant) +} + +)"; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + // Constant instructions have no name. The string will be parsed successfully + // but the constant names will not be exactly the same. +} + +TEST_F(HloParserTest, ConstantWithExp) { + const string original = R"(HloModule ConstantWithExp_module: + +ENTRY %ConstantWithExp.v4 () -> f32[] { + %constant.1 = f32[] constant(3e+2) +} + +)"; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + // The string will be parsed successfully but the output strings are not + // exactly the same, because "3e2" is parsed into value 300 and will be + // printed as "300". +} + +TEST_F(HloParserTest, Tuple) { + const string original = R"(HloModule EmptyTupleCreate_module: + +ENTRY %EmptyTupleCreate.v1 () -> () { + %tuple = () tuple() +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +} // namespace +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h new file mode 100644 index 0000000000..1f75e17c7f --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h @@ -0,0 +1,58 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ + +namespace xla { +namespace tools { + +// Defines different kinds of tokens in a hlo module string. +enum class TokKind { + // Markers + kEof, + kError, + + // Tokens with no info. + kEqual, // = + kComma, // , + kColon, // : + kLsquare, + kRsquare, // [ ] + kLbrace, + kRbrace, // { } + kLparen, + kRparen, // ( ) + + kArrow, // -> + + // Keywords + kw_HloModule, + kw_ENTRY, + kw_true, + kw_false, + + // Typed tokens. + kName, // %foo + kShape, // f32[2,3]{1,0} + kOpcode, // add + kInt, // 42 + kDecimal, // 4.2 +}; + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ -- GitLab From 47e92cfd08a230034268a1eeca625fd1e9908616 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 19 Oct 2017 14:23:03 -0700 Subject: [PATCH 169/573] `tf.py_func`: Handle NumPy arrays of np.object that hold unicode strings. This also fixes a bug affecting `tf.data.Dataset.from_generator()` on Python 3, where the generator yields Unicode (i.e. default) strings. PiperOrigin-RevId: 172798007 --- tensorflow/BUILD | 1 - .../compiler/xla/service/hlo_computation.cc | 4 +- tensorflow/compiler/xla/shape_util.cc | 45 +- tensorflow/compiler/xla/tools/parser/BUILD | 84 --- .../compiler/xla/tools/parser/README.md | 69 --- .../compiler/xla/tools/parser/hlo_lexer.cc | 270 ---------- .../compiler/xla/tools/parser/hlo_lexer.h | 108 ---- .../compiler/xla/tools/parser/hlo_parser.cc | 502 ------------------ .../compiler/xla/tools/parser/hlo_parser.h | 37 -- .../xla/tools/parser/hlo_parser_test.cc | 240 --------- .../compiler/xla/tools/parser/hlo_token.h | 58 -- 11 files changed, 16 insertions(+), 1402 deletions(-) delete mode 100644 tensorflow/compiler/xla/tools/parser/BUILD delete mode 100644 tensorflow/compiler/xla/tools/parser/README.md delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.cc delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.h delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.cc delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.h delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_token.h diff --git a/tensorflow/BUILD b/tensorflow/BUILD index d5c56cdc18..e351037abb 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -340,7 +340,6 @@ filegroup( "//tensorflow/compiler/xla/service/llvm_ir:all_files", "//tensorflow/compiler/xla/tests:all_files", "//tensorflow/compiler/xla/tools:all_files", - "//tensorflow/compiler/xla/tools/parser:all_files", "//tensorflow/contrib:all_files", "//tensorflow/contrib/all_reduce:all_files", "//tensorflow/contrib/android:all_files", diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 51ead753f0..9b3104eaac 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -373,8 +373,8 @@ string HloComputation::ToString(int nested_level) const { for (int i = 0; i < nested_level; i++) { s << " "; } - s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) - << " {\n"; + s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) + << " { \n"; for (const HloInstruction* instruction : MakeInstructionPostOrder()) { for (int i = 0; i < nested_level; i++) { s << " "; diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index af583bed62..8e16056b23 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -102,32 +102,6 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { return true; } -// Constructs and returns the new shape with the given minor_to_major order in -// its Layout. -StatusOr MakeShapeWithLayoutInternal( - PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, - tensorflow::gtl::ArraySlice minor_to_major) { - if (dimensions.size() != minor_to_major.size()) { - return InvalidArgument("Dimensions size is %ld, but layout size is %ld.", - dimensions.size(), minor_to_major.size()); - } - if (element_type == OPAQUE || element_type == TUPLE) { - return InvalidArgument("Unsupported element type: %s", - PrimitiveType_Name(element_type).c_str()); - } - Shape shape = ShapeUtil::MakeShape(element_type, dimensions); - auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); - min2maj->Clear(); - for (int64 value : minor_to_major) { - min2maj->Add(value); - } - if (!shape.has_layout()) { - return InvalidArgument("Shape has no layout."); - } - TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape)); - return shape; -} - } // namespace /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) { @@ -178,8 +152,16 @@ StatusOr MakeShapeWithLayoutInternal( /* static */ Shape ShapeUtil::MakeShapeWithLayout( PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, tensorflow::gtl::ArraySlice minor_to_major) { - return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major) - .ValueOrDie(); + CHECK_EQ(dimensions.size(), minor_to_major.size()); + Shape shape = MakeShape(element_type, dimensions); + auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); + min2maj->Clear(); + for (int64 value : minor_to_major) { + min2maj->Add(value); + } + DCHECK(shape.has_layout()); + TF_DCHECK_OK(ValidateShape(shape)); + return shape; } /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout( @@ -517,10 +499,11 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { // Extract the layout minor-to-major and set it. TF_ASSIGN_OR_RETURN(std::vector min2maj, comma_list_to_int64s(layout_string)); - TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal( - primitive_type, dimensions, min2maj)); + TF_RET_CHECK(dimensions.size() == min2maj.size()); + result = + ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj); } - TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result)); + TF_DCHECK_OK(ShapeUtil::ValidateShape(result)); return std::move(result); } diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD deleted file mode 100644 index c84ca9fc83..0000000000 --- a/tensorflow/compiler/xla/tools/parser/BUILD +++ /dev/null @@ -1,84 +0,0 @@ -# Build file for the Hlo parser. - -licenses(["notice"]) # Apache 2.0 - -package( - default_visibility = [":friends"], -) - -package_group( - name = "friends", - includes = [ - "//tensorflow/compiler/xla:friends", - ], -) - -# Filegroup used to collect source files for dependency checking. -filegroup( - name = "c_srcs", - data = glob([ - "**/*.cc", - "**/*.h", - ]), -) - -load("//tensorflow:tensorflow.bzl", "tf_cc_test") - -cc_library( - name = "hlo_lexer", - srcs = ["hlo_lexer.cc"], - hdrs = [ - "hlo_lexer.h", - "hlo_token.h", - ], - deps = [ - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:util", - "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/service:hlo", - "//tensorflow/core:lib", - "//tensorflow/core:regexp_internal", - ], -) - -cc_library( - name = "hlo_parser", - srcs = ["hlo_parser.cc"], - hdrs = ["hlo_parser.h"], - deps = [ - ":hlo_lexer", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:util", - "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/service:hlo", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - ], -) - -tf_cc_test( - name = "hlo_parser_test", - size = "small", - srcs = ["hlo_parser_test.cc"], - deps = [ - ":hlo_parser", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - -# ----------------------------------------------------------------------------- - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md deleted file mode 100644 index a334bc2b29..0000000000 --- a/tensorflow/compiler/xla/tools/parser/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# HloModule string syntax - -TODO: Support subcomputations (for fusion, reduce, while, ...). - -TODO: Support ops that require extra attributes, e.g. dimensions, strides. - -```yacc -hlo_module - : 'HloModule' name computation - ; - -computation - : 'ENTRY' name param_list '->' shape instruction_list - ; - -instruction_list - : '{' instruction_list1 '}' - ; -instruction_list1 - : instruction - | instruction_list1 instruction - ; -instruction - : name '=' shape opcode operands - ; - -operands - : '(' operands1 ')' - ; -operands1 - : /*empty*/ - | operand - | operands1 ',' operand - ; -operand - : shape name - ; - -param_list - : '(' param_list1 ')' - ; -param_list1 - : /*empty*/ - | param - | param_list1 ',' param - ; -param - : name shape - ; - -shape - : shape_val_ - | '(' tuple_elements ')' - ; -tuple_elements - : /*empty*/ - | shape (',' shape)* - ; - -name - : identifier ':' - | '%' identifier - ; - -identifier - : [a-zA-Z_][a-zA-Z0-9_.-]* - ; - -``` diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc deleted file mode 100644 index 3e84ffcbd2..0000000000 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc +++ /dev/null @@ -1,270 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" - -#include - -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/util.h" -#include "tensorflow/core/lib/gtl/optional.h" -#include "tensorflow/core/lib/strings/numbers.h" -#include "tensorflow/core/platform/regexp.h" - -namespace xla { -namespace tools { - -using tensorflow::StringPiece; - -namespace { - -constexpr int kEOF = -1; -constexpr int kError = -2; - -// [a-zA-Z0-9_.-] -bool IsIdentifierChar(char c) { - return isalnum(static_cast(c)) || c == '-' || c == '.' || - c == '_'; -} - -} // namespace - -int HloLexer::GetNextChar() { - int current_char = PeekCurrentChar(); - if (current_char != kEOF && current_char != kError) { - current_ptr_++; - } - return current_char; -} - -int HloLexer::PeekCurrentChar() const { - if (current_ptr_ == buf_.end()) { - return kEOF; - } - char current_char = *current_ptr_; - if (current_char == 0) { - // '\0' should not appear in the middle of the string. - return kError; - } - return static_cast(current_char); -} - -bool HloLexer::CanDereference(const char* ptr) const { - return ptr < buf_.end() && ptr >= buf_.begin(); -} - -StringPiece HloLexer::StringPieceFromPointers(const char* begin, - const char* end) const { - CHECK(begin <= end); - CHECK(begin == buf_.end() || CanDereference(begin)); - CHECK(end == buf_.end() || CanDereference(end)); - return StringPiece(begin, end - begin); -} - -tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers( - const char* begin, const char* end) const { - CHECK(begin <= end); - CHECK(begin == buf_.end() || CanDereference(begin)); - CHECK(end == buf_.end() || CanDereference(end)); - return tensorflow::RegexpStringPiece(begin, end - begin); -} - -TokKind HloLexer::LexToken() { - while (true) { - token_start_ = current_ptr_; - - int current_char = GetNextChar(); - switch (current_char) { - default: - // [a-zA-Z_] - if (isalpha(static_cast(current_char)) || - current_char == '_') { - return LexIdentifier(); - } - return TokKind::kError; - case kEOF: - // Hit the end of the input buffer. - return TokKind::kEof; - case kError: - // Hit an invalid character in the input buffer. - return TokKind::kError; - case ' ': - case '\t': - case '\n': - case '\r': - // Ignore whitespace. - continue; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '-': - if (current_char == '-' && PeekCurrentChar() == '>') { - current_ptr_++; - return TokKind::kArrow; - } - return LexDigitOrNegative(); - case '=': - return TokKind::kEqual; - case ',': - return TokKind::kComma; - case '%': - return LexPercent(); - case ':': - return TokKind::kColon; - case '[': - return TokKind::kLsquare; - case ']': - return TokKind::kRsquare; - case '{': - return TokKind::kLbrace; - case '}': - return TokKind::kRbrace; - case '(': - return TokKind::kLparen; - case ')': - return TokKind::kRparen; - } - } -} - -// Lex a shape, name, keyword, or opcode. -// shape ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})? -// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*: -// keyword ::= HloModule, ENTRY, ... -// opcode ::= add, greater-than, ... -TokKind HloLexer::LexIdentifier() { - { - auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); - // 'consumable' will be advanced iff its prefix matches the pattern. - static LazyRE2 shape_pattern = { - R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"}; - if (RE2::Consume(&consumable, *shape_pattern)) { - auto status_or_shape = ShapeUtil::ParseShapeString( - StringPieceFromPointers(token_start_, consumable.begin())); - if (status_or_shape.ok()) { - // This is a shape string. - shape_val_ = status_or_shape.ValueOrDie(); - current_ptr_ = consumable.begin(); - return TokKind::kShape; - } - } - } - - while (IsIdentifierChar(PeekCurrentChar())) { - current_ptr_++; - } - - // If followed by ':', it's a name. - if (PeekCurrentChar() == ':') { - str_val_.assign(token_start_, current_ptr_); - current_ptr_++; // skip ':' - return TokKind::kName; - } - - StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_); - - // See if this is a keyword. -#define KEYWORD(STR) \ - do { \ - if (identifier == #STR) { \ - return TokKind::kw_##STR; \ - } \ - } while (false) - - KEYWORD(true); - KEYWORD(false); - KEYWORD(HloModule); - KEYWORD(ENTRY); - -#undef KEYWORD - - // See if this is an opcode. - auto opcode = StringToHloOpcode(identifier.ToString()); - if (opcode.ok()) { - opcode_val_ = opcode.ValueOrDie(); - return TokKind::kOpcode; - } - - current_ptr_ = token_start_ + 1; - return TokKind::kError; -} - -// Lex names after a % character. -// name ::= [a-zA-Z_][a-zA-Z0-9_.-]* -TokKind HloLexer::LexPercent() { - const char* name_start = current_ptr_; - if (isalpha(static_cast(PeekCurrentChar())) || - PeekCurrentChar() == '_') { - current_ptr_++; - while (IsIdentifierChar(PeekCurrentChar())) { - current_ptr_++; - } - str_val_.assign(name_start, current_ptr_); - return TokKind::kName; - } - return TokKind::kError; -} - -// Lex integer and floating-point values. -// int [-]?[0-9]+ -// fp with exp [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+) -// fp without exp [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+) -TokKind HloLexer::LexDigitOrNegative() { - auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); - static LazyRE2 float_pattern = { - R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"}; - if (RE2::Consume(&consumable, *float_pattern)) { - current_ptr_ = consumable.begin(); - tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(), - &decimal_val_); - return TokKind::kDecimal; - } - - static LazyRE2 int_pattern = {R"([-]?\d+)"}; - if (RE2::Consume(&consumable, *int_pattern)) { - current_ptr_ = consumable.begin(); - tensorflow::strings::safe_strto64( - StringPieceFromPointers(token_start_, current_ptr_), &int64_val_); - return TokKind::kInt; - } - - return TokKind::kError; -} - -StringPiece HloLexer::GetCurrentLine() const { - const char* start = token_start_; - const char* end = current_ptr_; - if (!CanDereference(start) || !CanDereference(end)) { - return "LINE OUT OF RANGE"; - } - while (start > buf_.begin() && *start != '\n') { - start--; - } - while (end < buf_.end() && *end != '\n') { - end++; - } - return StringPieceFromPointers(start, end); -} - -} // namespace tools -} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h deleted file mode 100644 index 20278fd6cd..0000000000 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h +++ /dev/null @@ -1,108 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ -#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ - -#include - -#include "tensorflow/compiler/xla/service/hlo_opcode.h" -#include "tensorflow/compiler/xla/tools/parser/hlo_token.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/regexp.h" -#include "tensorflow/core/platform/types.h" - -namespace xla { -namespace tools { - -// Lexer for the HloModule::ToString() format text. -class HloLexer { - public: - explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) { - current_ptr_ = buf_.begin(); - } - - TokKind Lex() { return current_kind_ = LexToken(); } - TokKind GetKind() const { return current_kind_; } - string GetStrVal() const { - CHECK(GetKind() == TokKind::kName); - return str_val_; - } - Shape GetShapeVal() const { - CHECK(GetKind() == TokKind::kShape); - return shape_val_; - } - HloOpcode GetOpcodeVal() const { - CHECK(GetKind() == TokKind::kOpcode); - return opcode_val_; - } - int64 GetInt64Val() const { - CHECK(GetKind() == TokKind::kInt); - return int64_val_; - } - double GetDecimalVal() const { - CHECK(GetKind() == TokKind::kDecimal); - return decimal_val_; - } - - // Returns the line of text that is currently being lexed. - tensorflow::StringPiece GetCurrentLine() const; - - private: - // Returns the current character. If it's neither the end of input buffer nor - // an invalid character, moves the pointer forward. - int GetNextChar(); - - // Returns the current character. - int PeekCurrentChar() const; - - // Creates StringPiece with the given begin and end. Exits if the begin > end, - // or it's out of the range of the current buffer. - tensorflow::StringPiece StringPieceFromPointers(const char* begin, - const char* end) const; - tensorflow::RegexpStringPiece RegexpStringPieceFromPointers( - const char* begin, const char* end) const; - - // Returns true if the given ptr is dereferenceable within the range of the - // current buffer. - bool CanDereference(const char* ptr) const; - - TokKind LexToken(); - - TokKind LexIdentifier(); - TokKind LexPercent(); - TokKind LexShape(); - TokKind LexConstant(); - TokKind LexDigitOrNegative(); - - const tensorflow::StringPiece buf_; - const char* current_ptr_; - - // Information about the current token. - const char* token_start_; - TokKind current_kind_; - string str_val_; - Shape shape_val_; - HloOpcode opcode_val_; - int64 int64_val_; - double decimal_val_; -}; - -} // namespace tools -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc deleted file mode 100644 index 57700493e6..0000000000 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ /dev/null @@ -1,502 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" - -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/core/lib/gtl/map_util.h" -#include "tensorflow/core/lib/strings/strcat.h" - -namespace xla { -namespace tools { - -namespace { - -using tensorflow::StringPiece; -using tensorflow::strings::StrCat; - -// Parser for the HloModule::ToString() format text. -class HloParser { - public: - explicit HloParser(StringPiece str) : lexer_(str) {} - - // Runs the parser. Returns false if an error occurred. - bool Run(); - - // Returns the parsed HloModule. - std::unique_ptr ConsumeHloModule() { return std::move(module_); } - - // Returns the error information. - string GetError() const { return tensorflow::str_util::Join(error_, "\n"); } - - private: - // ParseXXX returns false if an error occurred. - bool ParseHloModule(); - bool ParseComputation(); - bool ParseInstructionList(HloComputation::Builder* builder); - bool ParseInstruction(HloComputation::Builder* builder); - bool ParseLiteral(std::unique_ptr* literal, const Shape& shape); - bool ParseOperands(std::vector* operands, - const int expected_size); - bool ParseParamList(); - bool ParseName(string* result); - bool ParseShape(Shape* result); - bool ParseOpcode(HloOpcode* result); - bool ParseInt64(int64* result); - bool ParseDecimal(double* result); - bool ParseBool(bool* result); - bool ParseToken(TokKind kind, const string& msg); - - // Logs the current parsing line and the given message. Always returns false. - bool TokenError(StringPiece msg); - - // If the current token is 'kind', eats it (i.e. lexes the next token) and - // returns true. - bool EatIfPresent(TokKind kind); - - // Adds the instruction to the pool. Returns false and emits an error if the - // instruction already exists. - bool AddInstruction(const string& name, HloInstruction* instruction); - - // The map from the instruction name to the instruction. This does not own the - // instructions. - std::unordered_map instruction_pool_; - - HloLexer lexer_; - std::unique_ptr module_; - std::vector error_; -}; - -bool HloParser::TokenError(StringPiece msg) { - error_.push_back( - StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg)); - return false; -} - -bool HloParser::Run() { - lexer_.Lex(); - return ParseHloModule(); -} - -// ::= 'HloModule' name computation -bool HloParser::ParseHloModule() { - if (lexer_.GetKind() != TokKind::kw_HloModule) { - return TokenError("expects HloModule"); - } - // Eat 'HloModule' - lexer_.Lex(); - - string name; - if (!ParseName(&name)) { - return false; - } - - module_ = MakeUnique(name); - - return ParseComputation(); -} - -// computation ::= 'ENTRY' name param_list '->' shape instruction_list -bool HloParser::ParseComputation() { - string name; - if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) { - return false; - } - auto builder = MakeUnique(name); - - Shape shape; - if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") || - !ParseShape(&shape) || !ParseInstructionList(builder.get())) { - return false; - } - module_->AddEntryComputation(builder->Build()); - return true; -} - -// instruction_list ::= '{' instruction_list1 '}' -// instruction_list1 ::= (instruction)+ -bool HloParser::ParseInstructionList(HloComputation::Builder* builder) { - if (!ParseToken(TokKind::kLbrace, - "expects '{' at the beginning of instruction list.")) { - return false; - } - do { - if (!ParseInstruction(builder)) { - return false; - } - } while (lexer_.GetKind() != TokKind::kRbrace); - return ParseToken(TokKind::kRbrace, - "expects '}' at the end of instruction list."); -} - -// instruction ::= name '=' shape opcode operands -bool HloParser::ParseInstruction(HloComputation::Builder* builder) { - string name; - Shape shape; - HloOpcode opcode; - std::vector operands; - if (!ParseName(&name) || - !ParseToken(TokKind::kEqual, "expects '=' in instruction") || - !ParseShape(&shape) || !ParseOpcode(&opcode)) { - return false; - } - switch (opcode) { - case HloOpcode::kParameter: { - int64 parameter_number; - return ParseToken(TokKind::kLparen, - "expects '(' before parameter number") && - ParseInt64(¶meter_number) && - ParseToken(TokKind::kRparen, - "expects ')' after parameter number") && - AddInstruction( - name, builder->AddInstruction(HloInstruction::CreateParameter( - parameter_number, shape, name))); - } - case HloOpcode::kConstant: { - std::unique_ptr literal; - return ParseToken(TokKind::kLparen, - "expects '(' before parameter number") && - ParseLiteral(&literal, shape) && - ParseToken(TokKind::kRparen, - "expects ')' after parameter number") && - AddInstruction( - name, builder->AddInstruction( - HloInstruction::CreateConstant(std::move(literal)))); - } - // Unary ops. - case HloOpcode::kAbs: - case HloOpcode::kRoundNearestAfz: - case HloOpcode::kBitcast: - case HloOpcode::kCeil: - case HloOpcode::kCopy: - case HloOpcode::kCos: - case HloOpcode::kExp: - case HloOpcode::kIsFinite: - case HloOpcode::kFloor: - case HloOpcode::kLog: - case HloOpcode::kNot: - case HloOpcode::kNegate: - case HloOpcode::kSign: - case HloOpcode::kSin: - case HloOpcode::kSort: - case HloOpcode::kTanh: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction(name, - builder->AddInstruction(HloInstruction::CreateUnary( - shape, opcode, operands[0]))); - } - // Binary ops. - case HloOpcode::kAdd: - case HloOpcode::kDivide: - case HloOpcode::kMultiply: - case HloOpcode::kSubtract: - case HloOpcode::kEq: - case HloOpcode::kGe: - case HloOpcode::kGt: - case HloOpcode::kLe: - case HloOpcode::kLt: - case HloOpcode::kNe: - case HloOpcode::kDot: - case HloOpcode::kMaximum: - case HloOpcode::kMinimum: - case HloOpcode::kPower: - case HloOpcode::kRemainder: - case HloOpcode::kAnd: - case HloOpcode::kOr: - case HloOpcode::kShiftLeft: - case HloOpcode::kShiftRightArithmetic: - case HloOpcode::kShiftRightLogical: { - return ParseOperands(&operands, /*expected_size=*/2) && - AddInstruction( - name, builder->AddInstruction(HloInstruction::CreateBinary( - shape, opcode, operands[0], operands[1]))); - } - // Ternary ops. - case HloOpcode::kClamp: - case HloOpcode::kSelect: { - return ParseOperands(&operands, /*expected_size=*/3) && - AddInstruction( - name, - builder->AddInstruction(HloInstruction::CreateTernary( - shape, opcode, operands[0], operands[1], operands[2]))); - } - // Other supported ops. - case HloOpcode::kConvert: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction( - name, builder->AddInstruction( - HloInstruction::CreateConvert(shape, operands[0]))); - } - case HloOpcode::kCrossReplicaSum: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction(name, builder->AddInstruction( - HloInstruction::CreateCrossReplicaSum( - shape, operands[0]))); - } - case HloOpcode::kReshape: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction( - name, builder->AddInstruction( - HloInstruction::CreateReshape(shape, operands[0]))); - } - case HloOpcode::kBroadcast: - case HloOpcode::kCall: - case HloOpcode::kCustomCall: - case HloOpcode::kConcatenate: - case HloOpcode::kReducePrecision: - case HloOpcode::kConvolution: - case HloOpcode::kGetTupleElement: - case HloOpcode::kMap: - case HloOpcode::kPad: - case HloOpcode::kReduce: - case HloOpcode::kReduceWindow: - case HloOpcode::kSelectAndScatter: - case HloOpcode::kReverse: - case HloOpcode::kRng: - case HloOpcode::kSlice: - case HloOpcode::kDynamicSlice: - case HloOpcode::kDynamicUpdateSlice: - case HloOpcode::kTranspose: - case HloOpcode::kTuple: - case HloOpcode::kWhile: - case HloOpcode::kFusion: - case HloOpcode::kBatchNormTraining: - case HloOpcode::kBatchNormInference: - case HloOpcode::kInfeed: - case HloOpcode::kOutfeed: - case HloOpcode::kBatchNormGrad: - case HloOpcode::kRecv: - case HloOpcode::kSend: - case HloOpcode::kUpdate: - case HloOpcode::kIndex: - case HloOpcode::kTrace: - return TokenError(StrCat("parsing not yet implemented for op: ", - HloOpcodeString(opcode))); - } -} - -bool HloParser::ParseLiteral(std::unique_ptr* literal, - const Shape& shape) { - switch (shape.element_type()) { - case PRED: - bool b; - if (!ParseBool(&b)) { - return false; - } - *literal = Literal::CreateR0(b); - return true; - case S32: - int64 i; - if (!ParseInt64(&i)) { - return false; - } - *literal = Literal::CreateR0(i); - return true; - case F32: - double d; - if (!ParseDecimal(&d)) { - return false; - } - *literal = Literal::CreateR0(d); - return true; - default: - return TokenError(StrCat("unsupported constant in shape: ", - ShapeUtil::HumanString(shape))); - } -} - -// operands ::= '(' operands1 ')' -// operands1 -// ::= /*empty*/ -// ::= operand (, operand)* -// operand ::= shape name -bool HloParser::ParseOperands(std::vector* operands, - const int expected_size) { - if (!ParseToken(TokKind::kLparen, - "expects '(' at the beginning of operands")) { - return false; - } - if (lexer_.GetKind() == TokKind::kRparen) { - // empty - } else { - do { - Shape shape; - string name; - if (!ParseShape(&shape) || !ParseName(&name)) { - return false; - } - HloInstruction* instruction = - tensorflow::gtl::FindPtrOrNull(instruction_pool_, name); - if (!instruction) { - return TokenError(StrCat("instruction does not exist: ", name)); - } - operands->push_back(instruction); - } while (EatIfPresent(TokKind::kComma)); - } - if (expected_size != operands->size()) { - return TokenError(StrCat("expects ", expected_size, " operands, but has ", - operands->size(), " operands")); - } - return ParseToken(TokKind::kRparen, "expects ')' at the end of operands"); -} - -// param_list ::= '(' param_list1 ')' -// param_list1 -// ::= /*empty*/ -// ::= param (',' param)* -// param ::= name shape -bool HloParser::ParseParamList() { - if (!ParseToken(TokKind::kLparen, - "expects '(' at the beginning of param list")) { - return false; - } - - if (lexer_.GetKind() == TokKind::kRparen) { - // empty - } else { - do { - Shape shape; - if (!ParseToken(TokKind::kName, "expects name in parameter") || - !ParseShape(&shape)) { - return false; - } - } while (EatIfPresent(TokKind::kComma)); - } - return ParseToken(TokKind::kRparen, "expects ')' at the end of param list"); -} - -// shape ::= shape_val_ -// shape ::= '(' tuple_elements ')' -// tuple_elements -// ::= /*empty*/ -// ::= shape (',' shape)* -bool HloParser::ParseShape(Shape* result) { - if (EatIfPresent(TokKind::kLparen)) { // Tuple - std::vector shapes; - if (lexer_.GetKind() == TokKind::kRparen) { - /*empty*/ - } else { - // shape (',' shape)* - do { - shapes.emplace_back(); - if (!ParseShape(&shapes.back())) { - return false; - } - } while (EatIfPresent(TokKind::kComma)); - } - *result = ShapeUtil::MakeTupleShape(shapes); - return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple."); - } - - if (lexer_.GetKind() != TokKind::kShape) { - return TokenError("expects shape"); - } - *result = lexer_.GetShapeVal(); - lexer_.Lex(); - return true; -} - -bool HloParser::ParseName(string* result) { - VLOG(1) << "ParseName"; - if (lexer_.GetKind() != TokKind::kName) { - return TokenError("expects name"); - } - *result = lexer_.GetStrVal(); - lexer_.Lex(); - return true; -} - -bool HloParser::ParseOpcode(HloOpcode* result) { - VLOG(1) << "ParseOpcode"; - if (lexer_.GetKind() != TokKind::kOpcode) { - return TokenError("expects opcode"); - } - *result = lexer_.GetOpcodeVal(); - lexer_.Lex(); - return true; -} - -bool HloParser::ParseInt64(int64* result) { - VLOG(1) << "ParseInt64"; - if (lexer_.GetKind() != TokKind::kInt) { - return TokenError("expects integer"); - } - *result = lexer_.GetInt64Val(); - lexer_.Lex(); - return true; -} - -bool HloParser::ParseDecimal(double* result) { - switch (lexer_.GetKind()) { - case TokKind::kDecimal: - *result = lexer_.GetDecimalVal(); - break; - case TokKind::kInt: - *result = static_cast(lexer_.GetInt64Val()); - break; - default: - return TokenError("expects decimal or integer"); - } - lexer_.Lex(); - return true; -} - -bool HloParser::ParseBool(bool* result) { - if (lexer_.GetKind() != TokKind::kw_true && - lexer_.GetKind() != TokKind::kw_false) { - return TokenError("expects true or false"); - } - *result = lexer_.GetKind() == TokKind::kw_true; - lexer_.Lex(); - return true; -} - -bool HloParser::ParseToken(TokKind kind, const string& msg) { - if (lexer_.GetKind() != kind) { - return TokenError(msg); - } - lexer_.Lex(); - return true; -} - -bool HloParser::EatIfPresent(TokKind kind) { - if (lexer_.GetKind() != kind) { - return false; - } - lexer_.Lex(); - return true; -} - -bool HloParser::AddInstruction(const string& name, - HloInstruction* instruction) { - auto result = instruction_pool_.insert({name, instruction}); - if (!result.second) { - return TokenError(StrCat("instruction already exists: ", name)); - } - return true; -} - -} // namespace - -StatusOr> Parse(StringPiece str) { - HloParser parser(str); - if (!parser.Run()) { - return InvalidArgument("Syntax error: %s", parser.GetError().c_str()); - } - return parser.ConsumeHloModule(); -} - -} // namespace tools -} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h deleted file mode 100644 index 9aaf18ef20..0000000000 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.h +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ -#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ - -#include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_module.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" - -namespace xla { -namespace tools { - -// The api of the hlo parser. Given a string in the HloModule::ToString() -// format, returns the parsed HloModule. -StatusOr> Parse(tensorflow::StringPiece str); - -} // namespace tools -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc deleted file mode 100644 index 4ecece3eac..0000000000 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc +++ /dev/null @@ -1,240 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" - -#include -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" - -namespace xla { -namespace tools { -namespace { - -struct TestData { - string test_name; - string module_string; -}; - -string TestDataToString(const ::testing::TestParamInfo& data) { - return data.param.test_name; -} - -std::vector CreateTestCases() { - // clang-format off - return std::vector({ -// ax + y -{ -"AxpyParam", -R"(HloModule axpy_module: - -ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { - %alpha = f32[2,4]{1,0} parameter(0) - %x = f32[2,4]{1,0} parameter(1) - %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x) - %y = f32[2,4]{1,0} parameter(2) - %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y) -} - -)" -}, -// pred constant -{ -"ConstantPred", -R"(HloModule constant_pred_module: - -ENTRY %constant_pred () -> pred[] { - %constant = pred[] constant(true) -} - -)" -}, -// s32 constant -{ -"ConstantS32", -R"(HloModule constant_s32_module: - -ENTRY %constant_s32 () -> s32[] { - %constant = s32[] constant(-42) -} - -)" -}, -// f32 constant, but the value is not a decimal -{ -"ConstantF32", R"(HloModule ConstantF32_module: - -ENTRY %ConstantF32.v4 () -> f32[] { - %constant = f32[] constant(42) -} - -)" -}, -// constant + constant -{ -"AddConstants", -R"(HloModule add_constants_module: - -ENTRY %add_constants () -> f32[] { - %constant = f32[] constant(3.14) - %add = f32[] add(f32[] %constant, f32[] %constant) -} - -)" -}, -// v1 > v2 ? v1 : v2 -{ -"SelectR1F32", -R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module: - -ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { - %v1 = f32[4]{0} parameter(0) - %v2 = f32[4]{0} parameter(1) - %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2) - %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) -} - -)" -} - }); - // clang-format on -} - -class HloParserTest : public ::testing::Test, - public ::testing::WithParamInterface { - protected: - void ExpectSuccess() { - const string& original = GetParam().module_string; - auto result = Parse(original); - TF_EXPECT_OK(result.status()); - EXPECT_EQ(original, result.ValueOrDie()->ToString()); - } -}; - -TEST_P(HloParserTest, Run) { ExpectSuccess(); } - -INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest, - ::testing::ValuesIn(CreateTestCases()), - TestDataToString); - -TEST_F(HloParserTest, Empty) { - const string original = ""; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -TEST_F(HloParserTest, Garbage) { - const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -TEST_F(HloParserTest, WrongOpcode) { - const string original = R"(HloModule wrong_opcode: - -ENTRY %blabla (x: f32[], y: f32[]) -> f32[] { - %x = f32[]{} parameter(0) - %y = f32[]{} parameter(1) - %le = pred[]{} le(f32[]{} %x, f32[]{} %y) -} - -)"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -TEST_F(HloParserTest, WrongShape) { - const string original = R"(HloModule wrong_opcode: - -ENTRY %blabla (x: g32[]) -> g32[] { - %x = g32[]{} parameter(0) -} - -)"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -TEST_F(HloParserTest, WrongOperandsSize) { - const string original = R"(HloModule wrong_opcode: - -ENTRY %blabla (x: f32[]) -> pred[] { - %x = f32[]{} parameter(0) - %eq = pred[]{} equal-to(f32[]{} %x) -} - -)"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -TEST_F(HloParserTest, OperandNotFound) { - const string original = R"(HloModule operand_not_found: -ENTRY %blabla (x: f32[]) -> pred[] { - %x = f32[]{} parameter(0) - %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y) -} -)"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -TEST_F(HloParserTest, MoreConstants) { - const string original = R"(HloModule SelectScalarS32True_module: - -ENTRY %SelectScalarS32True.v4 () -> s32[] { - %constant.2 = pred[] constant(true) - %constant.1 = s32[] constant(-42) - %constant = s32[] constant(42) - %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant) -} - -)"; - auto result = Parse(original); - TF_EXPECT_OK(result.status()); - // Constant instructions have no name. The string will be parsed successfully - // but the constant names will not be exactly the same. -} - -TEST_F(HloParserTest, ConstantWithExp) { - const string original = R"(HloModule ConstantWithExp_module: - -ENTRY %ConstantWithExp.v4 () -> f32[] { - %constant.1 = f32[] constant(3e+2) -} - -)"; - auto result = Parse(original); - TF_EXPECT_OK(result.status()); - // The string will be parsed successfully but the output strings are not - // exactly the same, because "3e2" is parsed into value 300 and will be - // printed as "300". -} - -TEST_F(HloParserTest, Tuple) { - const string original = R"(HloModule EmptyTupleCreate_module: - -ENTRY %EmptyTupleCreate.v1 () -> () { - %tuple = () tuple() -} - -)"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - -} // namespace -} // namespace tools -} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h deleted file mode 100644 index 1f75e17c7f..0000000000 --- a/tensorflow/compiler/xla/tools/parser/hlo_token.h +++ /dev/null @@ -1,58 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ -#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ - -namespace xla { -namespace tools { - -// Defines different kinds of tokens in a hlo module string. -enum class TokKind { - // Markers - kEof, - kError, - - // Tokens with no info. - kEqual, // = - kComma, // , - kColon, // : - kLsquare, - kRsquare, // [ ] - kLbrace, - kRbrace, // { } - kLparen, - kRparen, // ( ) - - kArrow, // -> - - // Keywords - kw_HloModule, - kw_ENTRY, - kw_true, - kw_false, - - // Typed tokens. - kName, // %foo - kShape, // f32[2,3]{1,0} - kOpcode, // add - kInt, // 42 - kDecimal, // 4.2 -}; - -} // namespace tools -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ -- GitLab From 6c074971ab80362954bea07ff2896cb91636b787 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 15:02:01 -0700 Subject: [PATCH 170/573] Add a recursive descent parser for the HloModule string. It constructs an HloModule object from a string printed by HloModule::ToString(). This is a initial stage. It currently supports: - unary, binary, ternary ops, and other ops that don't have extra attributes. - module with entry computation only. - simple cases for constant instruction. To make the parser simpler, this cl removes a whitespace and adds a '%' before the computation name in HloComputation::ToString(). Further steps will enable parsing subcomputations, more cases of constants, tuple, and ops that require extra attributes (e.g., broadcast dimensions, subcomputation). PiperOrigin-RevId: 172804214 --- tensorflow/BUILD | 1 + .../compiler/xla/service/hlo_computation.cc | 4 +- tensorflow/compiler/xla/shape_util.cc | 45 +- tensorflow/compiler/xla/tools/parser/BUILD | 84 +++ .../compiler/xla/tools/parser/README.md | 69 +++ .../compiler/xla/tools/parser/hlo_lexer.cc | 270 ++++++++++ .../compiler/xla/tools/parser/hlo_lexer.h | 108 ++++ .../compiler/xla/tools/parser/hlo_parser.cc | 502 ++++++++++++++++++ .../compiler/xla/tools/parser/hlo_parser.h | 37 ++ .../xla/tools/parser/hlo_parser_test.cc | 240 +++++++++ .../compiler/xla/tools/parser/hlo_token.h | 58 ++ 11 files changed, 1402 insertions(+), 16 deletions(-) create mode 100644 tensorflow/compiler/xla/tools/parser/BUILD create mode 100644 tensorflow/compiler/xla/tools/parser/README.md create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.cc create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.h create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.cc create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.h create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_token.h diff --git a/tensorflow/BUILD b/tensorflow/BUILD index e351037abb..d5c56cdc18 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -340,6 +340,7 @@ filegroup( "//tensorflow/compiler/xla/service/llvm_ir:all_files", "//tensorflow/compiler/xla/tests:all_files", "//tensorflow/compiler/xla/tools:all_files", + "//tensorflow/compiler/xla/tools/parser:all_files", "//tensorflow/contrib:all_files", "//tensorflow/contrib/all_reduce:all_files", "//tensorflow/contrib/android:all_files", diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 9b3104eaac..51ead753f0 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -373,8 +373,8 @@ string HloComputation::ToString(int nested_level) const { for (int i = 0; i < nested_level; i++) { s << " "; } - s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) - << " { \n"; + s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) + << " {\n"; for (const HloInstruction* instruction : MakeInstructionPostOrder()) { for (int i = 0; i < nested_level; i++) { s << " "; diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index 8e16056b23..af583bed62 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -102,6 +102,32 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { return true; } +// Constructs and returns the new shape with the given minor_to_major order in +// its Layout. +StatusOr MakeShapeWithLayoutInternal( + PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, + tensorflow::gtl::ArraySlice minor_to_major) { + if (dimensions.size() != minor_to_major.size()) { + return InvalidArgument("Dimensions size is %ld, but layout size is %ld.", + dimensions.size(), minor_to_major.size()); + } + if (element_type == OPAQUE || element_type == TUPLE) { + return InvalidArgument("Unsupported element type: %s", + PrimitiveType_Name(element_type).c_str()); + } + Shape shape = ShapeUtil::MakeShape(element_type, dimensions); + auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); + min2maj->Clear(); + for (int64 value : minor_to_major) { + min2maj->Add(value); + } + if (!shape.has_layout()) { + return InvalidArgument("Shape has no layout."); + } + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape)); + return shape; +} + } // namespace /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) { @@ -152,16 +178,8 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { /* static */ Shape ShapeUtil::MakeShapeWithLayout( PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, tensorflow::gtl::ArraySlice minor_to_major) { - CHECK_EQ(dimensions.size(), minor_to_major.size()); - Shape shape = MakeShape(element_type, dimensions); - auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); - min2maj->Clear(); - for (int64 value : minor_to_major) { - min2maj->Add(value); - } - DCHECK(shape.has_layout()); - TF_DCHECK_OK(ValidateShape(shape)); - return shape; + return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major) + .ValueOrDie(); } /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout( @@ -499,11 +517,10 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { // Extract the layout minor-to-major and set it. TF_ASSIGN_OR_RETURN(std::vector min2maj, comma_list_to_int64s(layout_string)); - TF_RET_CHECK(dimensions.size() == min2maj.size()); - result = - ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj); + TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal( + primitive_type, dimensions, min2maj)); } - TF_DCHECK_OK(ShapeUtil::ValidateShape(result)); + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result)); return std::move(result); } diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD new file mode 100644 index 0000000000..c84ca9fc83 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/BUILD @@ -0,0 +1,84 @@ +# Build file for the Hlo parser. + +licenses(["notice"]) # Apache 2.0 + +package( + default_visibility = [":friends"], +) + +package_group( + name = "friends", + includes = [ + "//tensorflow/compiler/xla:friends", + ], +) + +# Filegroup used to collect source files for dependency checking. +filegroup( + name = "c_srcs", + data = glob([ + "**/*.cc", + "**/*.h", + ]), +) + +load("//tensorflow:tensorflow.bzl", "tf_cc_test") + +cc_library( + name = "hlo_lexer", + srcs = ["hlo_lexer.cc"], + hdrs = [ + "hlo_lexer.h", + "hlo_token.h", + ], + deps = [ + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core:lib", + "//tensorflow/core:regexp_internal", + ], +) + +cc_library( + name = "hlo_parser", + srcs = ["hlo_parser.cc"], + hdrs = ["hlo_parser.h"], + deps = [ + ":hlo_lexer", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + ], +) + +tf_cc_test( + name = "hlo_parser_test", + size = "small", + srcs = ["hlo_parser_test.cc"], + deps = [ + ":hlo_parser", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + +# ----------------------------------------------------------------------------- + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md new file mode 100644 index 0000000000..a334bc2b29 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/README.md @@ -0,0 +1,69 @@ +# HloModule string syntax + +TODO: Support subcomputations (for fusion, reduce, while, ...). + +TODO: Support ops that require extra attributes, e.g. dimensions, strides. + +```yacc +hlo_module + : 'HloModule' name computation + ; + +computation + : 'ENTRY' name param_list '->' shape instruction_list + ; + +instruction_list + : '{' instruction_list1 '}' + ; +instruction_list1 + : instruction + | instruction_list1 instruction + ; +instruction + : name '=' shape opcode operands + ; + +operands + : '(' operands1 ')' + ; +operands1 + : /*empty*/ + | operand + | operands1 ',' operand + ; +operand + : shape name + ; + +param_list + : '(' param_list1 ')' + ; +param_list1 + : /*empty*/ + | param + | param_list1 ',' param + ; +param + : name shape + ; + +shape + : shape_val_ + | '(' tuple_elements ')' + ; +tuple_elements + : /*empty*/ + | shape (',' shape)* + ; + +name + : identifier ':' + | '%' identifier + ; + +identifier + : [a-zA-Z_][a-zA-Z0-9_.-]* + ; + +``` diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc new file mode 100644 index 0000000000..3e84ffcbd2 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc @@ -0,0 +1,270 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" + +#include + +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/gtl/optional.h" +#include "tensorflow/core/lib/strings/numbers.h" +#include "tensorflow/core/platform/regexp.h" + +namespace xla { +namespace tools { + +using tensorflow::StringPiece; + +namespace { + +constexpr int kEOF = -1; +constexpr int kError = -2; + +// [a-zA-Z0-9_.-] +bool IsIdentifierChar(char c) { + return isalnum(static_cast(c)) || c == '-' || c == '.' || + c == '_'; +} + +} // namespace + +int HloLexer::GetNextChar() { + int current_char = PeekCurrentChar(); + if (current_char != kEOF && current_char != kError) { + current_ptr_++; + } + return current_char; +} + +int HloLexer::PeekCurrentChar() const { + if (current_ptr_ == buf_.end()) { + return kEOF; + } + char current_char = *current_ptr_; + if (current_char == 0) { + // '\0' should not appear in the middle of the string. + return kError; + } + return static_cast(current_char); +} + +bool HloLexer::CanDereference(const char* ptr) const { + return ptr < buf_.end() && ptr >= buf_.begin(); +} + +StringPiece HloLexer::StringPieceFromPointers(const char* begin, + const char* end) const { + CHECK(begin <= end); + CHECK(begin == buf_.end() || CanDereference(begin)); + CHECK(end == buf_.end() || CanDereference(end)); + return StringPiece(begin, end - begin); +} + +tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers( + const char* begin, const char* end) const { + CHECK(begin <= end); + CHECK(begin == buf_.end() || CanDereference(begin)); + CHECK(end == buf_.end() || CanDereference(end)); + return tensorflow::RegexpStringPiece(begin, end - begin); +} + +TokKind HloLexer::LexToken() { + while (true) { + token_start_ = current_ptr_; + + int current_char = GetNextChar(); + switch (current_char) { + default: + // [a-zA-Z_] + if (isalpha(static_cast(current_char)) || + current_char == '_') { + return LexIdentifier(); + } + return TokKind::kError; + case kEOF: + // Hit the end of the input buffer. + return TokKind::kEof; + case kError: + // Hit an invalid character in the input buffer. + return TokKind::kError; + case ' ': + case '\t': + case '\n': + case '\r': + // Ignore whitespace. + continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (current_char == '-' && PeekCurrentChar() == '>') { + current_ptr_++; + return TokKind::kArrow; + } + return LexDigitOrNegative(); + case '=': + return TokKind::kEqual; + case ',': + return TokKind::kComma; + case '%': + return LexPercent(); + case ':': + return TokKind::kColon; + case '[': + return TokKind::kLsquare; + case ']': + return TokKind::kRsquare; + case '{': + return TokKind::kLbrace; + case '}': + return TokKind::kRbrace; + case '(': + return TokKind::kLparen; + case ')': + return TokKind::kRparen; + } + } +} + +// Lex a shape, name, keyword, or opcode. +// shape ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})? +// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*: +// keyword ::= HloModule, ENTRY, ... +// opcode ::= add, greater-than, ... +TokKind HloLexer::LexIdentifier() { + { + auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); + // 'consumable' will be advanced iff its prefix matches the pattern. + static LazyRE2 shape_pattern = { + R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"}; + if (RE2::Consume(&consumable, *shape_pattern)) { + auto status_or_shape = ShapeUtil::ParseShapeString( + StringPieceFromPointers(token_start_, consumable.begin())); + if (status_or_shape.ok()) { + // This is a shape string. + shape_val_ = status_or_shape.ValueOrDie(); + current_ptr_ = consumable.begin(); + return TokKind::kShape; + } + } + } + + while (IsIdentifierChar(PeekCurrentChar())) { + current_ptr_++; + } + + // If followed by ':', it's a name. + if (PeekCurrentChar() == ':') { + str_val_.assign(token_start_, current_ptr_); + current_ptr_++; // skip ':' + return TokKind::kName; + } + + StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_); + + // See if this is a keyword. +#define KEYWORD(STR) \ + do { \ + if (identifier == #STR) { \ + return TokKind::kw_##STR; \ + } \ + } while (false) + + KEYWORD(true); + KEYWORD(false); + KEYWORD(HloModule); + KEYWORD(ENTRY); + +#undef KEYWORD + + // See if this is an opcode. + auto opcode = StringToHloOpcode(identifier.ToString()); + if (opcode.ok()) { + opcode_val_ = opcode.ValueOrDie(); + return TokKind::kOpcode; + } + + current_ptr_ = token_start_ + 1; + return TokKind::kError; +} + +// Lex names after a % character. +// name ::= [a-zA-Z_][a-zA-Z0-9_.-]* +TokKind HloLexer::LexPercent() { + const char* name_start = current_ptr_; + if (isalpha(static_cast(PeekCurrentChar())) || + PeekCurrentChar() == '_') { + current_ptr_++; + while (IsIdentifierChar(PeekCurrentChar())) { + current_ptr_++; + } + str_val_.assign(name_start, current_ptr_); + return TokKind::kName; + } + return TokKind::kError; +} + +// Lex integer and floating-point values. +// int [-]?[0-9]+ +// fp with exp [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+) +// fp without exp [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+) +TokKind HloLexer::LexDigitOrNegative() { + auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); + static LazyRE2 float_pattern = { + R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"}; + if (RE2::Consume(&consumable, *float_pattern)) { + current_ptr_ = consumable.begin(); + tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(), + &decimal_val_); + return TokKind::kDecimal; + } + + static LazyRE2 int_pattern = {R"([-]?\d+)"}; + if (RE2::Consume(&consumable, *int_pattern)) { + current_ptr_ = consumable.begin(); + tensorflow::strings::safe_strto64( + StringPieceFromPointers(token_start_, current_ptr_), &int64_val_); + return TokKind::kInt; + } + + return TokKind::kError; +} + +StringPiece HloLexer::GetCurrentLine() const { + const char* start = token_start_; + const char* end = current_ptr_; + if (!CanDereference(start) || !CanDereference(end)) { + return "LINE OUT OF RANGE"; + } + while (start > buf_.begin() && *start != '\n') { + start--; + } + while (end < buf_.end() && *end != '\n') { + end++; + } + return StringPieceFromPointers(start, end); +} + +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h new file mode 100644 index 0000000000..20278fd6cd --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h @@ -0,0 +1,108 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ + +#include + +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_token.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/regexp.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { +namespace tools { + +// Lexer for the HloModule::ToString() format text. +class HloLexer { + public: + explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) { + current_ptr_ = buf_.begin(); + } + + TokKind Lex() { return current_kind_ = LexToken(); } + TokKind GetKind() const { return current_kind_; } + string GetStrVal() const { + CHECK(GetKind() == TokKind::kName); + return str_val_; + } + Shape GetShapeVal() const { + CHECK(GetKind() == TokKind::kShape); + return shape_val_; + } + HloOpcode GetOpcodeVal() const { + CHECK(GetKind() == TokKind::kOpcode); + return opcode_val_; + } + int64 GetInt64Val() const { + CHECK(GetKind() == TokKind::kInt); + return int64_val_; + } + double GetDecimalVal() const { + CHECK(GetKind() == TokKind::kDecimal); + return decimal_val_; + } + + // Returns the line of text that is currently being lexed. + tensorflow::StringPiece GetCurrentLine() const; + + private: + // Returns the current character. If it's neither the end of input buffer nor + // an invalid character, moves the pointer forward. + int GetNextChar(); + + // Returns the current character. + int PeekCurrentChar() const; + + // Creates StringPiece with the given begin and end. Exits if the begin > end, + // or it's out of the range of the current buffer. + tensorflow::StringPiece StringPieceFromPointers(const char* begin, + const char* end) const; + tensorflow::RegexpStringPiece RegexpStringPieceFromPointers( + const char* begin, const char* end) const; + + // Returns true if the given ptr is dereferenceable within the range of the + // current buffer. + bool CanDereference(const char* ptr) const; + + TokKind LexToken(); + + TokKind LexIdentifier(); + TokKind LexPercent(); + TokKind LexShape(); + TokKind LexConstant(); + TokKind LexDigitOrNegative(); + + const tensorflow::StringPiece buf_; + const char* current_ptr_; + + // Information about the current token. + const char* token_start_; + TokKind current_kind_; + string str_val_; + Shape shape_val_; + HloOpcode opcode_val_; + int64 int64_val_; + double decimal_val_; +}; + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc new file mode 100644 index 0000000000..57700493e6 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -0,0 +1,502 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" + +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/core/lib/gtl/map_util.h" +#include "tensorflow/core/lib/strings/strcat.h" + +namespace xla { +namespace tools { + +namespace { + +using tensorflow::StringPiece; +using tensorflow::strings::StrCat; + +// Parser for the HloModule::ToString() format text. +class HloParser { + public: + explicit HloParser(StringPiece str) : lexer_(str) {} + + // Runs the parser. Returns false if an error occurred. + bool Run(); + + // Returns the parsed HloModule. + std::unique_ptr ConsumeHloModule() { return std::move(module_); } + + // Returns the error information. + string GetError() const { return tensorflow::str_util::Join(error_, "\n"); } + + private: + // ParseXXX returns false if an error occurred. + bool ParseHloModule(); + bool ParseComputation(); + bool ParseInstructionList(HloComputation::Builder* builder); + bool ParseInstruction(HloComputation::Builder* builder); + bool ParseLiteral(std::unique_ptr* literal, const Shape& shape); + bool ParseOperands(std::vector* operands, + const int expected_size); + bool ParseParamList(); + bool ParseName(string* result); + bool ParseShape(Shape* result); + bool ParseOpcode(HloOpcode* result); + bool ParseInt64(int64* result); + bool ParseDecimal(double* result); + bool ParseBool(bool* result); + bool ParseToken(TokKind kind, const string& msg); + + // Logs the current parsing line and the given message. Always returns false. + bool TokenError(StringPiece msg); + + // If the current token is 'kind', eats it (i.e. lexes the next token) and + // returns true. + bool EatIfPresent(TokKind kind); + + // Adds the instruction to the pool. Returns false and emits an error if the + // instruction already exists. + bool AddInstruction(const string& name, HloInstruction* instruction); + + // The map from the instruction name to the instruction. This does not own the + // instructions. + std::unordered_map instruction_pool_; + + HloLexer lexer_; + std::unique_ptr module_; + std::vector error_; +}; + +bool HloParser::TokenError(StringPiece msg) { + error_.push_back( + StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg)); + return false; +} + +bool HloParser::Run() { + lexer_.Lex(); + return ParseHloModule(); +} + +// ::= 'HloModule' name computation +bool HloParser::ParseHloModule() { + if (lexer_.GetKind() != TokKind::kw_HloModule) { + return TokenError("expects HloModule"); + } + // Eat 'HloModule' + lexer_.Lex(); + + string name; + if (!ParseName(&name)) { + return false; + } + + module_ = MakeUnique(name); + + return ParseComputation(); +} + +// computation ::= 'ENTRY' name param_list '->' shape instruction_list +bool HloParser::ParseComputation() { + string name; + if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) { + return false; + } + auto builder = MakeUnique(name); + + Shape shape; + if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") || + !ParseShape(&shape) || !ParseInstructionList(builder.get())) { + return false; + } + module_->AddEntryComputation(builder->Build()); + return true; +} + +// instruction_list ::= '{' instruction_list1 '}' +// instruction_list1 ::= (instruction)+ +bool HloParser::ParseInstructionList(HloComputation::Builder* builder) { + if (!ParseToken(TokKind::kLbrace, + "expects '{' at the beginning of instruction list.")) { + return false; + } + do { + if (!ParseInstruction(builder)) { + return false; + } + } while (lexer_.GetKind() != TokKind::kRbrace); + return ParseToken(TokKind::kRbrace, + "expects '}' at the end of instruction list."); +} + +// instruction ::= name '=' shape opcode operands +bool HloParser::ParseInstruction(HloComputation::Builder* builder) { + string name; + Shape shape; + HloOpcode opcode; + std::vector operands; + if (!ParseName(&name) || + !ParseToken(TokKind::kEqual, "expects '=' in instruction") || + !ParseShape(&shape) || !ParseOpcode(&opcode)) { + return false; + } + switch (opcode) { + case HloOpcode::kParameter: { + int64 parameter_number; + return ParseToken(TokKind::kLparen, + "expects '(' before parameter number") && + ParseInt64(¶meter_number) && + ParseToken(TokKind::kRparen, + "expects ')' after parameter number") && + AddInstruction( + name, builder->AddInstruction(HloInstruction::CreateParameter( + parameter_number, shape, name))); + } + case HloOpcode::kConstant: { + std::unique_ptr literal; + return ParseToken(TokKind::kLparen, + "expects '(' before parameter number") && + ParseLiteral(&literal, shape) && + ParseToken(TokKind::kRparen, + "expects ')' after parameter number") && + AddInstruction( + name, builder->AddInstruction( + HloInstruction::CreateConstant(std::move(literal)))); + } + // Unary ops. + case HloOpcode::kAbs: + case HloOpcode::kRoundNearestAfz: + case HloOpcode::kBitcast: + case HloOpcode::kCeil: + case HloOpcode::kCopy: + case HloOpcode::kCos: + case HloOpcode::kExp: + case HloOpcode::kIsFinite: + case HloOpcode::kFloor: + case HloOpcode::kLog: + case HloOpcode::kNot: + case HloOpcode::kNegate: + case HloOpcode::kSign: + case HloOpcode::kSin: + case HloOpcode::kSort: + case HloOpcode::kTanh: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction(name, + builder->AddInstruction(HloInstruction::CreateUnary( + shape, opcode, operands[0]))); + } + // Binary ops. + case HloOpcode::kAdd: + case HloOpcode::kDivide: + case HloOpcode::kMultiply: + case HloOpcode::kSubtract: + case HloOpcode::kEq: + case HloOpcode::kGe: + case HloOpcode::kGt: + case HloOpcode::kLe: + case HloOpcode::kLt: + case HloOpcode::kNe: + case HloOpcode::kDot: + case HloOpcode::kMaximum: + case HloOpcode::kMinimum: + case HloOpcode::kPower: + case HloOpcode::kRemainder: + case HloOpcode::kAnd: + case HloOpcode::kOr: + case HloOpcode::kShiftLeft: + case HloOpcode::kShiftRightArithmetic: + case HloOpcode::kShiftRightLogical: { + return ParseOperands(&operands, /*expected_size=*/2) && + AddInstruction( + name, builder->AddInstruction(HloInstruction::CreateBinary( + shape, opcode, operands[0], operands[1]))); + } + // Ternary ops. + case HloOpcode::kClamp: + case HloOpcode::kSelect: { + return ParseOperands(&operands, /*expected_size=*/3) && + AddInstruction( + name, + builder->AddInstruction(HloInstruction::CreateTernary( + shape, opcode, operands[0], operands[1], operands[2]))); + } + // Other supported ops. + case HloOpcode::kConvert: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction( + name, builder->AddInstruction( + HloInstruction::CreateConvert(shape, operands[0]))); + } + case HloOpcode::kCrossReplicaSum: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction(name, builder->AddInstruction( + HloInstruction::CreateCrossReplicaSum( + shape, operands[0]))); + } + case HloOpcode::kReshape: { + return ParseOperands(&operands, /*expected_size=*/1) && + AddInstruction( + name, builder->AddInstruction( + HloInstruction::CreateReshape(shape, operands[0]))); + } + case HloOpcode::kBroadcast: + case HloOpcode::kCall: + case HloOpcode::kCustomCall: + case HloOpcode::kConcatenate: + case HloOpcode::kReducePrecision: + case HloOpcode::kConvolution: + case HloOpcode::kGetTupleElement: + case HloOpcode::kMap: + case HloOpcode::kPad: + case HloOpcode::kReduce: + case HloOpcode::kReduceWindow: + case HloOpcode::kSelectAndScatter: + case HloOpcode::kReverse: + case HloOpcode::kRng: + case HloOpcode::kSlice: + case HloOpcode::kDynamicSlice: + case HloOpcode::kDynamicUpdateSlice: + case HloOpcode::kTranspose: + case HloOpcode::kTuple: + case HloOpcode::kWhile: + case HloOpcode::kFusion: + case HloOpcode::kBatchNormTraining: + case HloOpcode::kBatchNormInference: + case HloOpcode::kInfeed: + case HloOpcode::kOutfeed: + case HloOpcode::kBatchNormGrad: + case HloOpcode::kRecv: + case HloOpcode::kSend: + case HloOpcode::kUpdate: + case HloOpcode::kIndex: + case HloOpcode::kTrace: + return TokenError(StrCat("parsing not yet implemented for op: ", + HloOpcodeString(opcode))); + } +} + +bool HloParser::ParseLiteral(std::unique_ptr* literal, + const Shape& shape) { + switch (shape.element_type()) { + case PRED: + bool b; + if (!ParseBool(&b)) { + return false; + } + *literal = Literal::CreateR0(b); + return true; + case S32: + int64 i; + if (!ParseInt64(&i)) { + return false; + } + *literal = Literal::CreateR0(i); + return true; + case F32: + double d; + if (!ParseDecimal(&d)) { + return false; + } + *literal = Literal::CreateR0(d); + return true; + default: + return TokenError(StrCat("unsupported constant in shape: ", + ShapeUtil::HumanString(shape))); + } +} + +// operands ::= '(' operands1 ')' +// operands1 +// ::= /*empty*/ +// ::= operand (, operand)* +// operand ::= shape name +bool HloParser::ParseOperands(std::vector* operands, + const int expected_size) { + if (!ParseToken(TokKind::kLparen, + "expects '(' at the beginning of operands")) { + return false; + } + if (lexer_.GetKind() == TokKind::kRparen) { + // empty + } else { + do { + Shape shape; + string name; + if (!ParseShape(&shape) || !ParseName(&name)) { + return false; + } + HloInstruction* instruction = + tensorflow::gtl::FindPtrOrNull(instruction_pool_, name); + if (!instruction) { + return TokenError(StrCat("instruction does not exist: ", name)); + } + operands->push_back(instruction); + } while (EatIfPresent(TokKind::kComma)); + } + if (expected_size != operands->size()) { + return TokenError(StrCat("expects ", expected_size, " operands, but has ", + operands->size(), " operands")); + } + return ParseToken(TokKind::kRparen, "expects ')' at the end of operands"); +} + +// param_list ::= '(' param_list1 ')' +// param_list1 +// ::= /*empty*/ +// ::= param (',' param)* +// param ::= name shape +bool HloParser::ParseParamList() { + if (!ParseToken(TokKind::kLparen, + "expects '(' at the beginning of param list")) { + return false; + } + + if (lexer_.GetKind() == TokKind::kRparen) { + // empty + } else { + do { + Shape shape; + if (!ParseToken(TokKind::kName, "expects name in parameter") || + !ParseShape(&shape)) { + return false; + } + } while (EatIfPresent(TokKind::kComma)); + } + return ParseToken(TokKind::kRparen, "expects ')' at the end of param list"); +} + +// shape ::= shape_val_ +// shape ::= '(' tuple_elements ')' +// tuple_elements +// ::= /*empty*/ +// ::= shape (',' shape)* +bool HloParser::ParseShape(Shape* result) { + if (EatIfPresent(TokKind::kLparen)) { // Tuple + std::vector shapes; + if (lexer_.GetKind() == TokKind::kRparen) { + /*empty*/ + } else { + // shape (',' shape)* + do { + shapes.emplace_back(); + if (!ParseShape(&shapes.back())) { + return false; + } + } while (EatIfPresent(TokKind::kComma)); + } + *result = ShapeUtil::MakeTupleShape(shapes); + return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple."); + } + + if (lexer_.GetKind() != TokKind::kShape) { + return TokenError("expects shape"); + } + *result = lexer_.GetShapeVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseName(string* result) { + VLOG(1) << "ParseName"; + if (lexer_.GetKind() != TokKind::kName) { + return TokenError("expects name"); + } + *result = lexer_.GetStrVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseOpcode(HloOpcode* result) { + VLOG(1) << "ParseOpcode"; + if (lexer_.GetKind() != TokKind::kOpcode) { + return TokenError("expects opcode"); + } + *result = lexer_.GetOpcodeVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseInt64(int64* result) { + VLOG(1) << "ParseInt64"; + if (lexer_.GetKind() != TokKind::kInt) { + return TokenError("expects integer"); + } + *result = lexer_.GetInt64Val(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseDecimal(double* result) { + switch (lexer_.GetKind()) { + case TokKind::kDecimal: + *result = lexer_.GetDecimalVal(); + break; + case TokKind::kInt: + *result = static_cast(lexer_.GetInt64Val()); + break; + default: + return TokenError("expects decimal or integer"); + } + lexer_.Lex(); + return true; +} + +bool HloParser::ParseBool(bool* result) { + if (lexer_.GetKind() != TokKind::kw_true && + lexer_.GetKind() != TokKind::kw_false) { + return TokenError("expects true or false"); + } + *result = lexer_.GetKind() == TokKind::kw_true; + lexer_.Lex(); + return true; +} + +bool HloParser::ParseToken(TokKind kind, const string& msg) { + if (lexer_.GetKind() != kind) { + return TokenError(msg); + } + lexer_.Lex(); + return true; +} + +bool HloParser::EatIfPresent(TokKind kind) { + if (lexer_.GetKind() != kind) { + return false; + } + lexer_.Lex(); + return true; +} + +bool HloParser::AddInstruction(const string& name, + HloInstruction* instruction) { + auto result = instruction_pool_.insert({name, instruction}); + if (!result.second) { + return TokenError(StrCat("instruction already exists: ", name)); + } + return true; +} + +} // namespace + +StatusOr> Parse(StringPiece str) { + HloParser parser(str); + if (!parser.Run()) { + return InvalidArgument("Syntax error: %s", parser.GetError().c_str()); + } + return parser.ConsumeHloModule(); +} + +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h new file mode 100644 index 0000000000..9aaf18ef20 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.h @@ -0,0 +1,37 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ + +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" + +namespace xla { +namespace tools { + +// The api of the hlo parser. Given a string in the HloModule::ToString() +// format, returns the parsed HloModule. +StatusOr> Parse(tensorflow::StringPiece str); + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc new file mode 100644 index 0000000000..4ecece3eac --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -0,0 +1,240 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" + +#include +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { +namespace tools { +namespace { + +struct TestData { + string test_name; + string module_string; +}; + +string TestDataToString(const ::testing::TestParamInfo& data) { + return data.param.test_name; +} + +std::vector CreateTestCases() { + // clang-format off + return std::vector({ +// ax + y +{ +"AxpyParam", +R"(HloModule axpy_module: + +ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { + %alpha = f32[2,4]{1,0} parameter(0) + %x = f32[2,4]{1,0} parameter(1) + %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x) + %y = f32[2,4]{1,0} parameter(2) + %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y) +} + +)" +}, +// pred constant +{ +"ConstantPred", +R"(HloModule constant_pred_module: + +ENTRY %constant_pred () -> pred[] { + %constant = pred[] constant(true) +} + +)" +}, +// s32 constant +{ +"ConstantS32", +R"(HloModule constant_s32_module: + +ENTRY %constant_s32 () -> s32[] { + %constant = s32[] constant(-42) +} + +)" +}, +// f32 constant, but the value is not a decimal +{ +"ConstantF32", R"(HloModule ConstantF32_module: + +ENTRY %ConstantF32.v4 () -> f32[] { + %constant = f32[] constant(42) +} + +)" +}, +// constant + constant +{ +"AddConstants", +R"(HloModule add_constants_module: + +ENTRY %add_constants () -> f32[] { + %constant = f32[] constant(3.14) + %add = f32[] add(f32[] %constant, f32[] %constant) +} + +)" +}, +// v1 > v2 ? v1 : v2 +{ +"SelectR1F32", +R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module: + +ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { + %v1 = f32[4]{0} parameter(0) + %v2 = f32[4]{0} parameter(1) + %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2) + %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) +} + +)" +} + }); + // clang-format on +} + +class HloParserTest : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + void ExpectSuccess() { + const string& original = GetParam().module_string; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + EXPECT_EQ(original, result.ValueOrDie()->ToString()); + } +}; + +TEST_P(HloParserTest, Run) { ExpectSuccess(); } + +INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest, + ::testing::ValuesIn(CreateTestCases()), + TestDataToString); + +TEST_F(HloParserTest, Empty) { + const string original = ""; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, Garbage) { + const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongOpcode) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: f32[], y: f32[]) -> f32[] { + %x = f32[]{} parameter(0) + %y = f32[]{} parameter(1) + %le = pred[]{} le(f32[]{} %x, f32[]{} %y) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongShape) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: g32[]) -> g32[] { + %x = g32[]{} parameter(0) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongOperandsSize) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: f32[]) -> pred[] { + %x = f32[]{} parameter(0) + %eq = pred[]{} equal-to(f32[]{} %x) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, OperandNotFound) { + const string original = R"(HloModule operand_not_found: +ENTRY %blabla (x: f32[]) -> pred[] { + %x = f32[]{} parameter(0) + %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y) +} +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, MoreConstants) { + const string original = R"(HloModule SelectScalarS32True_module: + +ENTRY %SelectScalarS32True.v4 () -> s32[] { + %constant.2 = pred[] constant(true) + %constant.1 = s32[] constant(-42) + %constant = s32[] constant(42) + %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant) +} + +)"; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + // Constant instructions have no name. The string will be parsed successfully + // but the constant names will not be exactly the same. +} + +TEST_F(HloParserTest, ConstantWithExp) { + const string original = R"(HloModule ConstantWithExp_module: + +ENTRY %ConstantWithExp.v4 () -> f32[] { + %constant.1 = f32[] constant(3e+2) +} + +)"; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + // The string will be parsed successfully but the output strings are not + // exactly the same, because "3e2" is parsed into value 300 and will be + // printed as "300". +} + +TEST_F(HloParserTest, Tuple) { + const string original = R"(HloModule EmptyTupleCreate_module: + +ENTRY %EmptyTupleCreate.v1 () -> () { + %tuple = () tuple() +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +} // namespace +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h new file mode 100644 index 0000000000..1f75e17c7f --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h @@ -0,0 +1,58 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ + +namespace xla { +namespace tools { + +// Defines different kinds of tokens in a hlo module string. +enum class TokKind { + // Markers + kEof, + kError, + + // Tokens with no info. + kEqual, // = + kComma, // , + kColon, // : + kLsquare, + kRsquare, // [ ] + kLbrace, + kRbrace, // { } + kLparen, + kRparen, // ( ) + + kArrow, // -> + + // Keywords + kw_HloModule, + kw_ENTRY, + kw_true, + kw_false, + + // Typed tokens. + kName, // %foo + kShape, // f32[2,3]{1,0} + kOpcode, // add + kInt, // 42 + kDecimal, // 4.2 +}; + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ -- GitLab From 2cd178ef5a4e5cac27b55729f0203c4864540063 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Thu, 19 Oct 2017 15:22:08 -0700 Subject: [PATCH 171/573] [XLA] Teach transpose folding how to transpose the LHS of convolutions This is now possible now that we have added the required fields to ConvolutionDimensionNumbers. PiperOrigin-RevId: 172807540 --- .../compiler/xla/service/transpose_folding.cc | 105 ++++++++++++------ .../xla/service/transpose_folding_test.cc | 28 +++-- 2 files changed, 89 insertions(+), 44 deletions(-) diff --git a/tensorflow/compiler/xla/service/transpose_folding.cc b/tensorflow/compiler/xla/service/transpose_folding.cc index 816c8a7485..8c2640adf5 100644 --- a/tensorflow/compiler/xla/service/transpose_folding.cc +++ b/tensorflow/compiler/xla/service/transpose_folding.cc @@ -58,14 +58,32 @@ TransposeFolding::OperandIndices CanFoldOperandsIntoConvolution( return {}; } - // We only support folding the RHS. - const int64 kRhsOperandIndex = 1; - auto& operand = *convolution.operand(kRhsOperandIndex); - if (operand.opcode() == HloOpcode::kTranspose && operand.user_count() == 1) { - return transposable_conv_operands(convolution, {kRhsOperandIndex}); + const ConvolutionDimensionNumbers& dnums = + convolution.convolution_dimension_numbers(); + + TransposeFolding::OperandIndices operand_set; + for (int64 i = 0; i < convolution.operand_count(); ++i) { + auto& operand = *convolution.operand(i); + if (operand.opcode() == HloOpcode::kTranspose && + operand.user_count() == 1) { + const auto& transpose_dimensions = operand.dimensions(); + // We can transpose the LHS so long as it doesn't move around spatial + // dimensions because ConvolutionDimensionNumbers doesn't have different + // fields for input and output spatial dimensions. + if (i == 0 && + std::any_of(dnums.spatial_dimensions().begin(), + dnums.spatial_dimensions().end(), + [&](const int64 spatial_dimension) { + return transpose_dimensions[spatial_dimension] != + spatial_dimension; + })) { + continue; + } + operand_set.push_back(i); + } } - return {}; + return transposable_conv_operands(convolution, operand_set); } using InstructionOperandsPair = @@ -98,40 +116,61 @@ bool FoldTransposeIntoDot(InstructionOperandsPair pair) { // Returns whether the module is changed. bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) { auto& convolution = *pair.first; - - // We only support fusing the RHS transpose into convolution. - // - // ConvolutionDimensionNumbers doesn't make enough of a distinction between - // the output and the activations. - // - // TODO(b/37125184): Support transposing the LHS too. - if (pair.second.size() != 1 || pair.second.front() != 1) { - return false; - } + auto& operand_indices = pair.second; const ConvolutionDimensionNumbers& dnums = convolution.convolution_dimension_numbers(); - HloInstruction& transpose = *convolution.mutable_operand(1); - CHECK_EQ(transpose.opcode(), HloOpcode::kTranspose); - const auto& transpose_dimensions = transpose.dimensions(); - HloInstruction& transpose_operand = *transpose.mutable_operand(0); - - // Everything remains the same except for the kernel dimension numbers. We - // need to apply the transpose permutation to the original shape to figure out - // what the new logical dimensions are. ConvolutionDimensionNumbers new_dnums = dnums; - new_dnums.set_kernel_input_feature_dimension( - transpose_dimensions[dnums.kernel_input_feature_dimension()]); - new_dnums.set_kernel_output_feature_dimension( - transpose_dimensions[dnums.kernel_output_feature_dimension()]); - for (auto& kernel_spatial_dimension : - *new_dnums.mutable_kernel_spatial_dimensions()) { - kernel_spatial_dimension = transpose_dimensions[kernel_spatial_dimension]; + + HloInstruction* new_lhs; + const int64 kLhsIdx = 0; + if (std::find(operand_indices.begin(), operand_indices.end(), kLhsIdx) != + operand_indices.end()) { + HloInstruction& transpose = *convolution.mutable_operand(kLhsIdx); + const auto& transpose_dimensions = transpose.dimensions(); + HloInstruction& transpose_operand = *transpose.mutable_operand(0); + + // Everything remains the same except for the input/output dimension + // numbers. We need to apply the transpose permutation to the original shape + // to figure out what the new logical dimensions are. + new_dnums.set_input_batch_dimension( + transpose_dimensions[dnums.input_batch_dimension()]); + new_dnums.set_input_feature_dimension( + transpose_dimensions[dnums.input_feature_dimension()]); + for (const auto& spatial_dimension : dnums.spatial_dimensions()) { + CHECK_EQ(spatial_dimension, transpose_dimensions[spatial_dimension]); + } + new_lhs = &transpose_operand; + } else { + new_lhs = convolution.mutable_operand(kLhsIdx); + } + + HloInstruction* new_rhs; + const int64 kRhsIdx = 1; + if (std::find(operand_indices.begin(), operand_indices.end(), kRhsIdx) != + operand_indices.end()) { + HloInstruction& transpose = *convolution.mutable_operand(kRhsIdx); + const auto& transpose_dimensions = transpose.dimensions(); + HloInstruction& transpose_operand = *transpose.mutable_operand(0); + + // Everything remains the same except for the kernel dimension numbers. We + // need to apply the transpose permutation to the original shape to figure + // out what the new logical dimensions are. + new_dnums.set_kernel_input_feature_dimension( + transpose_dimensions[dnums.kernel_input_feature_dimension()]); + new_dnums.set_kernel_output_feature_dimension( + transpose_dimensions[dnums.kernel_output_feature_dimension()]); + for (auto& kernel_spatial_dimension : + *new_dnums.mutable_kernel_spatial_dimensions()) { + kernel_spatial_dimension = transpose_dimensions[kernel_spatial_dimension]; + } + new_rhs = &transpose_operand; + } else { + new_rhs = convolution.mutable_operand(kRhsIdx); } auto new_conv = HloInstruction::CreateConvolve( - convolution.shape(), convolution.mutable_operand(0), &transpose_operand, - convolution.window(), new_dnums); + convolution.shape(), new_lhs, new_rhs, convolution.window(), new_dnums); TF_CHECK_OK(convolution.parent()->ReplaceWithNewInstruction( &convolution, std::move(new_conv))); diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc index a6161b4646..00462f9be1 100644 --- a/tensorflow/compiler/xla/service/transpose_folding_test.cc +++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc @@ -313,8 +313,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeRhs) { new_conv->convolution_dimension_numbers().kernel_spatial_dimensions(1)); } -// Test that a transpose of the activations does not get folded into -// convolution. +// Test that a transpose of the activations gets folded into convolution. TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) { auto builder = HloComputation::Builder("entry_computation"); HloInstruction* x = builder.AddInstruction(HloInstruction::CreateParameter( @@ -348,18 +347,25 @@ TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) { module.AddEntryComputation(builder.Build(conv)); FoldTranspose(&module); - // Instructions after folding: transpose_x, y, and the convolution. + // Instructions after folding: x, y, and the convolution. std::unordered_set instruction_set( entry_computation->instructions().begin(), entry_computation->instructions().end()); - CHECK_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation."; - CHECK_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation."; - CHECK_EQ(1, instruction_set.erase(transpose_x)) - << "transpose_x is not in entry_computation."; - CHECK_EQ(1, instruction_set.erase(conv)) - << "transpose_x is not in entry_computation."; - CHECK_EQ(0, instruction_set.size()) - << "entry_computation should contain exactly 4 instructions."; + EXPECT_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation."; + EXPECT_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation."; + EXPECT_EQ(1, instruction_set.size()) + << "entry_computation should contain exactly 3 instructions."; + HloInstruction* new_conv = *instruction_set.begin(); + EXPECT_EQ(HloOpcode::kConvolution, new_conv->opcode()); + EXPECT_EQ(dnums.input_feature_dimension(), + new_conv->convolution_dimension_numbers().input_batch_dimension()); + EXPECT_EQ( + dnums.input_batch_dimension(), + new_conv->convolution_dimension_numbers().input_feature_dimension()); + EXPECT_EQ(dnums.spatial_dimensions(0), + new_conv->convolution_dimension_numbers().spatial_dimensions(0)); + EXPECT_EQ(dnums.spatial_dimensions(1), + new_conv->convolution_dimension_numbers().spatial_dimensions(1)); } } // namespace -- GitLab From f080052284a4a39113051fb1178d91365e9872a8 Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Thu, 19 Oct 2017 15:27:52 -0700 Subject: [PATCH 172/573] Move text_classification_character_rnn from .contrib utils to .core utils. Also removes sklearn comparison. PiperOrigin-RevId: 172808535 --- .../text_classification_character_rnn.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tensorflow/examples/learn/text_classification_character_rnn.py b/tensorflow/examples/learn/text_classification_character_rnn.py index 1fc9388a1a..86adc056ad 100644 --- a/tensorflow/examples/learn/text_classification_character_rnn.py +++ b/tensorflow/examples/learn/text_classification_character_rnn.py @@ -30,7 +30,6 @@ import sys import numpy as np import pandas -from sklearn import metrics import tensorflow as tf FLAGS = None @@ -46,8 +45,8 @@ def char_rnn_model(features, labels, mode): byte_vectors = tf.one_hot(features[CHARS_FEATURE], 256, 1., 0.) byte_list = tf.unstack(byte_vectors, axis=1) - cell = tf.contrib.rnn.GRUCell(HIDDEN_SIZE) - _, encoding = tf.contrib.rnn.static_rnn(cell, byte_list, dtype=tf.float32) + cell = tf.nn.rnn_cell.GRUCell(HIDDEN_SIZE) + _, encoding = tf.nn.static_rnn(cell, byte_list, dtype=tf.float32) logits = tf.layers.dense(encoding, MAX_LABEL, activation=None) @@ -98,28 +97,20 @@ def main(unused_argv): train_input_fn = tf.estimator.inputs.numpy_input_fn( x={CHARS_FEATURE: x_train}, y=y_train, - batch_size=len(x_train), + batch_size=128, num_epochs=None, shuffle=True) classifier.train(input_fn=train_input_fn, steps=100) - # Predict. + # Eval. test_input_fn = tf.estimator.inputs.numpy_input_fn( x={CHARS_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) - predictions = classifier.predict(input_fn=test_input_fn) - y_predicted = np.array(list(p['class'] for p in predictions)) - y_predicted = y_predicted.reshape(np.array(y_test).shape) - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - - # Score with tensorflow. scores = classifier.evaluate(input_fn=test_input_fn) - print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) + print('Accuracy: {0:f}'.format(scores['accuracy'])) if __name__ == '__main__': -- GitLab From bc93dcbd9f7b445c5f6f0d1c8f597324d412a76a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 16:00:31 -0700 Subject: [PATCH 173/573] Fix precision/recall test. Precision and Recall have as the numerator TP: true positives. The labels generated in the test were only negative, and hence the test passed before because all updates were 0. PiperOrigin-RevId: 172812994 --- .../metrics/python/ops/metric_ops_test.py | 58 +++++++++---------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py index cc0ad155fa..f288fceef6 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py @@ -1101,7 +1101,7 @@ class StreamingPrecisionTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) precision, update_op = metrics.streaming_precision(predictions, labels) with self.test_session() as sess: @@ -1265,7 +1265,7 @@ class StreamingRecallTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) recall, update_op = metrics.streaming_recall(predictions, labels) with self.test_session() as sess: @@ -1388,7 +1388,7 @@ class StreamingFPRTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) fpr, update_op = metrics.streaming_false_positive_rate( predictions, labels) @@ -1516,7 +1516,7 @@ class StreamingFNRTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) fnr, update_op = metrics.streaming_false_negative_rate( predictions, labels) @@ -1737,7 +1737,7 @@ class StreamingAUCTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) auc, update_op = metrics.streaming_auc(predictions, labels) with self.test_session() as sess: @@ -2009,7 +2009,7 @@ class StreamingSpecificityAtSensitivityTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) specificity, update_op = metrics.streaming_specificity_at_sensitivity( predictions, labels, sensitivity=0.7) @@ -2271,7 +2271,7 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) thresholds = [0, 0.5, 1.0] prec, prec_op = metrics.streaming_precision_at_thresholds(predictions, labels, @@ -2282,12 +2282,14 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): with self.test_session() as sess: sess.run(variables.local_variables_initializer()) - # Run several updates, then verify idempotency. - sess.run([prec_op, rec_op]) + # Run several updates. + for _ in range(10): + sess.run([prec_op, rec_op]) + + # Then verify idempotency. initial_prec = prec.eval() initial_rec = rec.eval() for _ in range(10): - sess.run([prec_op, rec_op]) self.assertAllClose(initial_prec, prec.eval()) self.assertAllClose(initial_rec, rec.eval()) @@ -2361,14 +2363,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.streaming_recall_at_thresholds( predictions, labels, thresholds, weights=weights) - [prec_low, prec_high] = array_ops.split( - value=prec, num_or_size_splits=2, axis=0) - prec_low = array_ops.reshape(prec_low, shape=()) - prec_high = array_ops.reshape(prec_high, shape=()) - [rec_low, rec_high] = array_ops.split( - value=rec, num_or_size_splits=2, axis=0) - rec_low = array_ops.reshape(rec_low, shape=()) - rec_high = array_ops.reshape(rec_high, shape=()) + prec_low = prec[0] + prec_high = prec[1] + rec_low = rec[0] + rec_high = rec[1] sess.run(variables.local_variables_initializer()) sess.run([prec_op, rec_op]) @@ -2391,14 +2389,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.streaming_recall_at_thresholds( predictions, labels, thresholds, weights=weights) - [prec_low, prec_high] = array_ops.split( - value=prec, num_or_size_splits=2, axis=0) - prec_low = array_ops.reshape(prec_low, shape=()) - prec_high = array_ops.reshape(prec_high, shape=()) - [rec_low, rec_high] = array_ops.split( - value=rec, num_or_size_splits=2, axis=0) - rec_low = array_ops.reshape(rec_low, shape=()) - rec_high = array_ops.reshape(rec_high, shape=()) + prec_low = prec[0] + prec_high = prec[1] + rec_low = rec[0] + rec_high = rec[1] sess.run(variables.local_variables_initializer()) sess.run([prec_op, rec_op]) @@ -2420,10 +2414,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.streaming_recall_at_thresholds(predictions, labels, thresholds) - [prec_low, prec_high] = array_ops.split( - value=prec, num_or_size_splits=2, axis=0) - [rec_low, rec_high] = array_ops.split( - value=rec, num_or_size_splits=2, axis=0) + prec_low = prec[0] + prec_high = prec[1] + rec_low = rec[0] + rec_high = rec[1] sess.run(variables.local_variables_initializer()) sess.run([prec_op, rec_op]) @@ -2562,7 +2556,7 @@ class StreamingFPRThresholdsTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) thresholds = [0, 0.5, 1.0] fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( predictions, labels, thresholds) @@ -2794,7 +2788,7 @@ class StreamingFNRThresholdsTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) thresholds = [0, 0.5, 1.0] fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( predictions, labels, thresholds) -- GitLab From 7a253f3da99c3692d464a8dd95d8280d4cd8973a Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Thu, 19 Oct 2017 16:16:29 -0700 Subject: [PATCH 174/573] Fix random_forest_mnist.py and eliminate a contrib.learn reference to skcompat. PiperOrigin-RevId: 172815173 --- .../examples/learn/random_forest_mnist.py | 65 ++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/tensorflow/examples/learn/random_forest_mnist.py b/tensorflow/examples/learn/random_forest_mnist.py index 3c09990ea1..72c935cdae 100644 --- a/tensorflow/examples/learn/random_forest_mnist.py +++ b/tensorflow/examples/learn/random_forest_mnist.py @@ -1,4 +1,4 @@ - # Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,18 +21,14 @@ import argparse import sys import tempfile -# pylint: disable=g-backslash-continuation -from tensorflow.contrib.learn.python.learn\ - import metric_spec -from tensorflow.contrib.learn.python.learn.estimators\ - import estimator -from tensorflow.contrib.tensor_forest.client\ - import eval_metrics -from tensorflow.contrib.tensor_forest.client\ - import random_forest -from tensorflow.contrib.tensor_forest.python\ - import tensor_forest +import numpy + +from tensorflow.contrib.learn.python.learn import metric_spec +from tensorflow.contrib.tensor_forest.client import eval_metrics +from tensorflow.contrib.tensor_forest.client import random_forest +from tensorflow.contrib.tensor_forest.python import tensor_forest from tensorflow.examples.tutorials.mnist import input_data +from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.platform import app FLAGS = None @@ -41,16 +37,15 @@ FLAGS = None def build_estimator(model_dir): """Build an estimator.""" params = tensor_forest.ForestHParams( - num_classes=10, num_features=784, - num_trees=FLAGS.num_trees, max_nodes=FLAGS.max_nodes) + num_classes=10, + num_features=784, + num_trees=FLAGS.num_trees, + max_nodes=FLAGS.max_nodes) graph_builder_class = tensor_forest.RandomForestGraphs if FLAGS.use_training_loss: graph_builder_class = tensor_forest.TrainingLossForest - # Use the SKCompat wrapper, which gives us a convenient way to split - # in-memory data like MNIST into batches. - return estimator.SKCompat(random_forest.TensorForestEstimator( - params, graph_builder_class=graph_builder_class, - model_dir=model_dir)) + return random_forest.TensorForestEstimator( + params, graph_builder_class=graph_builder_class, model_dir=model_dir) def train_and_eval(): @@ -62,18 +57,30 @@ def train_and_eval(): mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=False) - est.fit(x=mnist.train.images, y=mnist.train.labels, - batch_size=FLAGS.batch_size) + train_input_fn = numpy_io.numpy_input_fn( + x={'images': mnist.train.images}, + y=mnist.train.labels.astype(numpy.int32), + batch_size=FLAGS.batch_size, + num_epochs=None, + shuffle=True) + est.fit(input_fn=train_input_fn, steps=None) metric_name = 'accuracy' - metric = {metric_name: - metric_spec.MetricSpec( - eval_metrics.get_metric(metric_name), - prediction_key=eval_metrics.get_prediction_key(metric_name))} - - results = est.score(x=mnist.test.images, y=mnist.test.labels, - batch_size=FLAGS.batch_size, - metrics=metric) + metric = { + metric_name: + metric_spec.MetricSpec( + eval_metrics.get_metric(metric_name), + prediction_key=eval_metrics.get_prediction_key(metric_name)) + } + + test_input_fn = numpy_io.numpy_input_fn( + x={'images': mnist.test.images}, + y=mnist.test.labels.astype(numpy.int32), + num_epochs=1, + batch_size=FLAGS.batch_size, + shuffle=False) + + results = est.evaluate(input_fn=test_input_fn, metrics=metric) for key in sorted(results): print('%s: %s' % (key, results[key])) -- GitLab From 60a03dfc7dbde7acf58ffaeef897eb3ebb98603f Mon Sep 17 00:00:00 2001 From: Michael Case Date: Thu, 19 Oct 2017 16:18:46 -0700 Subject: [PATCH 175/573] Move s3 file system support from contrib/ to core/platform/. PiperOrigin-RevId: 172815422 --- tensorflow/BUILD | 2 +- tensorflow/contrib/makefile/Makefile | 1 + tensorflow/core/platform/default/build_config.bzl | 2 +- tensorflow/{contrib => core/platform}/s3/BUILD | 0 tensorflow/{contrib => core/platform}/s3/s3_crypto.cc | 2 +- tensorflow/{contrib => core/platform}/s3/s3_crypto.h | 0 tensorflow/{contrib => core/platform}/s3/s3_file_system.cc | 4 ++-- tensorflow/{contrib => core/platform}/s3/s3_file_system.h | 0 .../{contrib => core/platform}/s3/s3_file_system_test.cc | 2 +- 9 files changed, 7 insertions(+), 6 deletions(-) rename tensorflow/{contrib => core/platform}/s3/BUILD (100%) rename tensorflow/{contrib => core/platform}/s3/s3_crypto.cc (98%) rename tensorflow/{contrib => core/platform}/s3/s3_crypto.h (100%) rename tensorflow/{contrib => core/platform}/s3/s3_file_system.cc (99%) rename tensorflow/{contrib => core/platform}/s3/s3_file_system.h (100%) rename tensorflow/{contrib => core/platform}/s3/s3_file_system_test.cc (99%) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index d5c56cdc18..d7d6d5fc77 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -414,7 +414,6 @@ filegroup( "//tensorflow/contrib/remote_fused_graph/pylib:all_files", "//tensorflow/contrib/resampler:all_files", "//tensorflow/contrib/rnn:all_files", - "//tensorflow/contrib/s3:all_files", "//tensorflow/contrib/saved_model:all_files", "//tensorflow/contrib/saved_model/cc/saved_model:all_files", "//tensorflow/contrib/seq2seq:all_files", @@ -468,6 +467,7 @@ filegroup( "//tensorflow/core/platform/cloud:all_files", "//tensorflow/core/platform/default/build_config:all_files", "//tensorflow/core/platform/hadoop:all_files", + "//tensorflow/core/platform/s3:all_files", "//tensorflow/core/profiler:all_files", "//tensorflow/core/profiler/internal:all_files", "//tensorflow/core/profiler/internal/advisor:all_files", diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index be7c790ee9..3dcff3d4a3 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -502,6 +502,7 @@ $(wildcard tensorflow/core/platform/google/*) \ $(wildcard tensorflow/core/platform/google/*/*) \ $(wildcard tensorflow/core/platform/jpeg.*) \ $(wildcard tensorflow/core/platform/png.*) \ +$(wildcard tensorflow/core/platform/s3/*) \ $(wildcard tensorflow/core/platform/stream_executor.*) \ $(wildcard tensorflow/core/platform/windows/*) \ $(wildcard tensorflow/core/user_ops/*.cu.cc) \ diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 2c14ea917c..e4518a8e2f 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -467,7 +467,7 @@ def tf_additional_core_deps(): "//conditions:default": [], }) + select({ "//tensorflow:with_s3_support": [ - "//tensorflow/contrib/s3:s3_file_system", + "//tensorflow/core/platform/s3:s3_file_system", ], "//conditions:default": [], }) diff --git a/tensorflow/contrib/s3/BUILD b/tensorflow/core/platform/s3/BUILD similarity index 100% rename from tensorflow/contrib/s3/BUILD rename to tensorflow/core/platform/s3/BUILD diff --git a/tensorflow/contrib/s3/s3_crypto.cc b/tensorflow/core/platform/s3/s3_crypto.cc similarity index 98% rename from tensorflow/contrib/s3/s3_crypto.cc rename to tensorflow/core/platform/s3/s3_crypto.cc index 1450384dc0..14bbed19a5 100644 --- a/tensorflow/contrib/s3/s3_crypto.cc +++ b/tensorflow/core/platform/s3/s3_crypto.cc @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/s3/s3_crypto.h" +#include "tensorflow/core/platform/s3/s3_crypto.h" #include #include diff --git a/tensorflow/contrib/s3/s3_crypto.h b/tensorflow/core/platform/s3/s3_crypto.h similarity index 100% rename from tensorflow/contrib/s3/s3_crypto.h rename to tensorflow/core/platform/s3/s3_crypto.h diff --git a/tensorflow/contrib/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc similarity index 99% rename from tensorflow/contrib/s3/s3_file_system.cc rename to tensorflow/core/platform/s3/s3_file_system.cc index daced83145..51c85592bf 100644 --- a/tensorflow/contrib/s3/s3_file_system.cc +++ b/tensorflow/core/platform/s3/s3_file_system.cc @@ -12,10 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/s3/s3_file_system.h" -#include "tensorflow/contrib/s3/s3_crypto.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/s3/s3_file_system.h" +#include "tensorflow/core/platform/s3/s3_crypto.h" #include #include diff --git a/tensorflow/contrib/s3/s3_file_system.h b/tensorflow/core/platform/s3/s3_file_system.h similarity index 100% rename from tensorflow/contrib/s3/s3_file_system.h rename to tensorflow/core/platform/s3/s3_file_system.h diff --git a/tensorflow/contrib/s3/s3_file_system_test.cc b/tensorflow/core/platform/s3/s3_file_system_test.cc similarity index 99% rename from tensorflow/contrib/s3/s3_file_system_test.cc rename to tensorflow/core/platform/s3/s3_file_system_test.cc index 949281fad4..0b42f5fcec 100644 --- a/tensorflow/contrib/s3/s3_file_system_test.cc +++ b/tensorflow/core/platform/s3/s3_file_system_test.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/s3/s3_file_system.h" +#include "tensorflow/core/platform/s3/s3_file_system.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/gtl/stl_util.h" -- GitLab From d88cccebc7f61078d775d26f4714a06bc4002fcf Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Thu, 19 Oct 2017 16:20:06 -0700 Subject: [PATCH 176/573] Rename SNAPPY to TF_USE_SNAPPY This way there's less risk of it conflicting with downstream BUILD rules. PiperOrigin-RevId: 172815580 --- tensorflow/contrib/cmake/external/snappy.cmake | 2 +- tensorflow/core/BUILD | 2 +- tensorflow/core/platform/posix/port.cc | 8 ++++---- tensorflow/core/platform/windows/port.cc | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/cmake/external/snappy.cmake b/tensorflow/contrib/cmake/external/snappy.cmake index a35d8654fb..2d2451521c 100644 --- a/tensorflow/contrib/cmake/external/snappy.cmake +++ b/tensorflow/contrib/cmake/external/snappy.cmake @@ -47,4 +47,4 @@ ExternalProject_Add(snappy ) # actually enables snappy in the source code -add_definitions(-DSNAPPY) \ No newline at end of file +add_definitions(-DTF_USE_SNAPPY) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 5ab84fec5b..d198a796a7 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1410,7 +1410,7 @@ cc_library( hdrs = LIB_INTERNAL_PUBLIC_HEADERS, copts = tf_copts(), defines = tf_additional_lib_defines() + [ - "SNAPPY", + "TF_USE_SNAPPY", ] + tf_additional_verbs_lib_defines() + tf_additional_mpi_lib_defines() + tf_additional_gdr_lib_defines(), diff --git a/tensorflow/core/platform/posix/port.cc b/tensorflow/core/platform/posix/port.cc index 3b17bac808..93a59348c8 100644 --- a/tensorflow/core/platform/posix/port.cc +++ b/tensorflow/core/platform/posix/port.cc @@ -29,7 +29,7 @@ limitations under the License. #include #include #include -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY #include "snappy.h" #endif #if (defined(__APPLE__) && defined(__MACH__)) || defined(__FreeBSD__) @@ -126,7 +126,7 @@ void AdjustFilenameForLogging(string* filename) { } bool Snappy_Compress(const char* input, size_t length, string* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); @@ -139,7 +139,7 @@ bool Snappy_Compress(const char* input, size_t length, string* output) { bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; @@ -147,7 +147,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length, } bool Snappy_Uncompress(const char* input, size_t length, char* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::RawUncompress(input, length, output); #else return false; diff --git a/tensorflow/core/platform/windows/port.cc b/tensorflow/core/platform/windows/port.cc index 85b53e07c4..e327d53949 100644 --- a/tensorflow/core/platform/windows/port.cc +++ b/tensorflow/core/platform/windows/port.cc @@ -20,7 +20,7 @@ limitations under the License. #include #include #include -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY #include "snappy.h" #endif @@ -118,7 +118,7 @@ void AdjustFilenameForLogging(string* filename) { } bool Snappy_Compress(const char* input, size_t length, string* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); @@ -131,7 +131,7 @@ bool Snappy_Compress(const char* input, size_t length, string* output) { bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; @@ -139,7 +139,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length, } bool Snappy_Uncompress(const char* input, size_t length, char* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::RawUncompress(input, length, output); #else return false; -- GitLab From f2250bfe85b59c9fba128aad9993417eca711d75 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 16:24:56 -0700 Subject: [PATCH 177/573] Replace http://mirror.bazel.build with https://mirror.bazel.build PiperOrigin-RevId: 172816169 --- WORKSPACE | 2 +- tensorflow/contrib/cmake/external/cub.cmake | 2 +- tensorflow/contrib/cmake/external/gif.cmake | 2 +- tensorflow/contrib/cmake/external/jpeg.cmake | 2 +- tensorflow/contrib/cmake/external/lmdb.cmake | 2 +- .../contrib/makefile/download_dependencies.sh | 8 +- tensorflow/workspace.bzl | 90 +++++++++---------- 7 files changed, 54 insertions(+), 54 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 1bf1069f88..b40913801b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,7 +5,7 @@ http_archive( sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257", strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", # 2017-08-28 ], ) diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake index d98579d207..e03026b1b0 100644 --- a/tensorflow/contrib/cmake/external/cub.cmake +++ b/tensorflow/contrib/cmake/external/cub.cmake @@ -14,7 +14,7 @@ # ============================================================================== include (ExternalProject) -set(cub_URL http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip) +set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip) set(cub_HASH SHA256=b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe) set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) diff --git a/tensorflow/contrib/cmake/external/gif.cmake b/tensorflow/contrib/cmake/external/gif.cmake index 5cb719b878..3d53c51fff 100644 --- a/tensorflow/contrib/cmake/external/gif.cmake +++ b/tensorflow/contrib/cmake/external/gif.cmake @@ -15,7 +15,7 @@ include (ExternalProject) set(gif_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/gif_archive/giflib-5.1.4/) -set(gif_URL http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz) +set(gif_URL https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz) set(gif_HASH SHA256=34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1) set(gif_INSTALL ${CMAKE_BINARY_DIR}/gif/install) set(gif_BUILD ${CMAKE_BINARY_DIR}/gif/src/gif) diff --git a/tensorflow/contrib/cmake/external/jpeg.cmake b/tensorflow/contrib/cmake/external/jpeg.cmake index 058f554b8f..d9a165e856 100644 --- a/tensorflow/contrib/cmake/external/jpeg.cmake +++ b/tensorflow/contrib/cmake/external/jpeg.cmake @@ -15,7 +15,7 @@ include (ExternalProject) set(jpeg_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/jpeg_archive) -set(jpeg_URL http://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz) +set(jpeg_URL https://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz) set(jpeg_HASH SHA256=3a753ea48d917945dd54a2d97de388aa06ca2eb1066cbfdc6652036349fe05a7) set(jpeg_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jpeg/src/jpeg) set(jpeg_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/jpeg/install) diff --git a/tensorflow/contrib/cmake/external/lmdb.cmake b/tensorflow/contrib/cmake/external/lmdb.cmake index 28ec833bab..79971b7cfc 100644 --- a/tensorflow/contrib/cmake/external/lmdb.cmake +++ b/tensorflow/contrib/cmake/external/lmdb.cmake @@ -15,7 +15,7 @@ include (ExternalProject) set(lmdb_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/lmdb) -set(lmdb_URL http://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz) +set(lmdb_URL https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz) set(lmdb_HASH SHA256=108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326) set(lmdb_BUILD ${CMAKE_BINARY_DIR}/lmdb/src/lmdb) set(lmdb_INSTALL ${CMAKE_BINARY_DIR}/lmdb/install) diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh index 39c89628d9..f0b9658e3d 100755 --- a/tensorflow/contrib/makefile/download_dependencies.sh +++ b/tensorflow/contrib/makefile/download_dependencies.sh @@ -20,11 +20,11 @@ DOWNLOADS_DIR=tensorflow/contrib/makefile/downloads BZL_FILE_PATH=tensorflow/workspace.bzl EIGEN_URL="$(grep -o 'http.*bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" -GEMMLOWP_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" +GEMMLOWP_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz" -NSYNC_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" -PROTOBUF_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" -RE2_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" +NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" +PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" +RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64, diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 54559edbea..a863aa18dd 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -157,7 +157,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): mkl_repository( name = "mkl", urls = [ - "http://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz", + "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz", # "https://github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz", ], sha256 = "57ba56c4c243f403ff78f417ff854ef50b9eddf4a610a917b7c95e7fa8553a4b", @@ -174,7 +174,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "mkl_dnn", urls = [ "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", - "http://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", + "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", ], sha256 = "0d529ad4c49dc799e6df07c2b88b115d0668735da15fb3b3862d28d33fa68165", strip_prefix = "mkl-dnn-b01e3a55a07be62172e713bcd2644c5176360212", @@ -185,7 +185,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "eigen_archive", urls = [ "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", - "http://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", + "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", ], sha256 = "61d8b6fc4279dd1dda986fb1677d15e3d641c07a3ea5abe255790b1f0c0c14e9", strip_prefix = "eigen-eigen-429aa5254200", @@ -198,7 +198,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "970285762565c7890c6c087d262b0a18286e7d0384f13a37786d8521773bc969", strip_prefix = "tools-0e906ebc527eab1cdbf7adabff5b474da9562e9f/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf", urls = [ - "http://mirror.bazel.build/github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz", + "https://mirror.bazel.build/github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz", # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz", ], ) @@ -206,7 +206,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "libxsmm_archive", urls = [ - "http://mirror.bazel.build/github.com/hfp/libxsmm/archive/1.8.1.tar.gz", + "https://mirror.bazel.build/github.com/hfp/libxsmm/archive/1.8.1.tar.gz", # "https://github.com/hfp/libxsmm/archive/1.8.1.tar.gz", ], sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce", @@ -222,7 +222,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "ortools_archive", urls = [ - "http://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz", + "https://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz", # "https://github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz", ], sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755", @@ -233,7 +233,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_googlesource_code_re2", urls = [ - "http://mirror.bazel.build/github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz", + "https://mirror.bazel.build/github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz", # "https://github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz", ], sha256 = "bd63550101e056427c9e7ff12a408c1c8b74e9803f393ca916b2926fc2c4906f", @@ -243,7 +243,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "gemmlowp", urls = [ - "http://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip" + "https://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip" # "https://github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip", ], sha256 = "dd2557072bde12141419cb8320a9c25e6ec41a8ae53c2ac78c076a347bb46d9d", @@ -253,7 +253,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "farmhash_archive", urls = [ - "http://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz", + "https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz", # "https://github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz", ], sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0", @@ -269,7 +269,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "highwayhash", urls = [ - "http://mirror.bazel.build/github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz", + "https://mirror.bazel.build/github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz", # "https://github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz", ], sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9", @@ -280,7 +280,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "nasm", urls = [ - "http://mirror.bazel.build/www.nasm.us/pub/nasm/releasebuilds/2.12.02/nasm-2.12.02.tar.bz2", + "https://mirror.bazel.build/www.nasm.us/pub/nasm/releasebuilds/2.12.02/nasm-2.12.02.tar.bz2", "http://pkgs.fedoraproject.org/repo/pkgs/nasm/nasm-2.12.02.tar.bz2/d15843c3fb7db39af80571ee27ec6fad/nasm-2.12.02.tar.bz2", ], sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324", @@ -291,7 +291,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "jpeg", urls = [ - "http://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", + "https://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", # "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", ], sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77", @@ -303,7 +303,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "png_archive", urls = [ - "http://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.2.53.tar.gz", + "https://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.2.53.tar.gz", # "https://github.com/glennrp/libpng/archive/v1.2.53.tar.gz", ], sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2", @@ -314,7 +314,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "sqlite_archive", urls = [ - "http://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", + "https://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", "http://www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", ], sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4", @@ -325,7 +325,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "gif_archive", urls = [ - "http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz", + "https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz", "http://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz", ], sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1", @@ -336,7 +336,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "six_archive", urls = [ - "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz", + "https://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz", "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz", ], sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", @@ -347,7 +347,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "org_python_pypi_backports_weakref", urls = [ - "http://mirror.bazel.build/pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz", + "https://mirror.bazel.build/pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz", "https://pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz", ], sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892", @@ -358,7 +358,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "com_github_andreif_codegen", urls = [ - "http://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz", + "https://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz", # "https://github.com/andreif/codegen/archive/1.0.tar.gz", ], sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee", @@ -371,7 +371,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): licenses = ["notice"], # Python 2.0 sha256_urls = { "b5556e921715ddb9242c076cae3963f483aa47266c5e37ea4c187f77cc79501c": [ - "http://mirror.bazel.build/docs.python.org/2.7/_sources/license.txt", + "https://mirror.bazel.build/docs.python.org/2.7/_sources/license.txt", "https://docs.python.org/2.7/_sources/license.txt", ], }, @@ -387,7 +387,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): patched_http_archive( name = "protobuf_archive", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz", ], sha256 = "e178a25c52efcb6b05988bdbeace4c0d3f2d2fe5b46696d1d9898875c3803d6a", strip_prefix = "protobuf-b04e5cba356212e4e8c66c61bbe0c3a20537c5b9", @@ -410,7 +410,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_google_protobuf", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", @@ -420,7 +420,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_google_protobuf_cc", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", @@ -429,7 +429,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "nsync", urls = [ - "http://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz", + "https://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz", # "https://github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz", ], sha256 = "7dd8ca49319f77e8226cd020a9210a525f88ac26e7041c59c95418223a1cdf55", @@ -439,7 +439,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_google_googletest", urls = [ - "http://mirror.bazel.build/github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip", + "https://mirror.bazel.build/github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip", # "https://github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip", ], sha256 = "9cbca84c4256bed17df2c8f4d00c912c19d247c11c9ba6647cd6dd5b5c996b8d", @@ -449,7 +449,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_github_gflags_gflags", urls = [ - "http://mirror.bazel.build/github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz", + "https://mirror.bazel.build/github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz", # "https://github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz", ], sha256 = "4d222fab8f1ede4709cdff417d15a1336f862d7334a81abf76d09c15ecf9acd1", @@ -465,7 +465,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "pcre", sha256 = "ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7", urls = [ - "http://mirror.bazel.build/ftp.exim.org/pub/pcre/pcre-8.39.tar.gz", + "https://mirror.bazel.build/ftp.exim.org/pub/pcre/pcre-8.39.tar.gz", "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz", ], strip_prefix = "pcre-8.39", @@ -476,7 +476,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "swig", sha256 = "58a475dbbd4a4d7075e5fe86d4e54c9edde39847cdb96a3053d87cb64a23a453", urls = [ - "http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", + "https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", "http://ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", ], @@ -488,7 +488,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "curl", sha256 = "ff3e80c1ca6a068428726cd7dd19037a47cc538ce58ef61c59587191039b2ca6", urls = [ - "http://mirror.bazel.build/curl.haxx.se/download/curl-7.49.1.tar.gz", + "https://mirror.bazel.build/curl.haxx.se/download/curl-7.49.1.tar.gz", "https://curl.haxx.se/download/curl-7.49.1.tar.gz", ], strip_prefix = "curl-7.49.1", @@ -518,7 +518,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): patched_http_archive( name = "grpc", urls = [ - "http://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz", + "https://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz", # "https://github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz", ], sha256 = "2004635e6a078acfac8ffa71738397796be4f8fb72f572cc44ecee5d99511d9f", @@ -542,7 +542,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "linenoise", sha256 = "7f51f45887a3d31b4ce4fa5965210a5e64637ceac12720cfce7954d6a2e812f7", urls = [ - "http://mirror.bazel.build/github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz", + "https://mirror.bazel.build/github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz", # "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz", ], strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3", @@ -554,7 +554,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "llvm", urls = [ - "http://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz", "https://github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz", ], sha256 = "caab6d7978e6771cb4e9b5b89607c5370de8aa642913c6c14e892468194c94e4", @@ -566,7 +566,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "lmdb", urls = [ - "http://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz", + "https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz", # "https://github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz", ], sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326", @@ -577,7 +577,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "jsoncpp_git", urls = [ - "http://mirror.bazel.build/github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz", + "https://mirror.bazel.build/github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz", # "https://github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz", ], sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2", @@ -593,7 +593,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): patched_http_archive( name = "boringssl", urls = [ - "http://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz", + "https://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz", # "https://github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz", # 2017-07-07 ], sha256 = "02f5950f93c4fd3691771c07c9d04cf2999ab01383ff99da345249e93b0fcfb2", @@ -605,7 +605,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "zlib_archive", urls = [ - "http://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz", + "https://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz", "http://zlib.net/fossils/zlib-1.2.8.tar.gz", ], sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d", @@ -621,7 +621,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "fft2d", urls = [ - "http://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz", + "https://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz", "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz", ], sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296", @@ -631,7 +631,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "snappy", urls = [ - "http://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz", + "https://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz", # "https://github.com/google/snappy/archive/1.1.4.tar.gz", ], sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94", @@ -643,7 +643,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "nccl_archive", urls = [ - "http://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", + "https://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", # "https://github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", ], sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176", @@ -668,7 +668,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "junit", jar_sha256 = "59721f0805e223d84b90677887d9ff567dc534d7c502ca903c0c2b17f05c116a", jar_urls = [ - "http://mirror.bazel.build/repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar", + "https://mirror.bazel.build/repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar", "http://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar", "http://maven.ibiblio.org/maven2/junit/junit/4.12/junit-4.12.jar", ], @@ -681,7 +681,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "org_hamcrest_core", jar_sha256 = "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9", jar_urls = [ - "http://mirror.bazel.build/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", + "https://mirror.bazel.build/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", "http://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", "http://maven.ibiblio.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", ], @@ -692,7 +692,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "jemalloc", urls = [ - "http://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", + "https://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", # "https://github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", ], sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8", @@ -704,7 +704,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "com_google_pprof", urls = [ - "http://mirror.bazel.build/github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz", + "https://mirror.bazel.build/github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz", # "https://github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz", ], sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4", @@ -715,7 +715,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "cub_archive", urls = [ - "http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip", + "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip", # "https://github.com/NVlabs/cub/archive/1.7.3.zip", ], sha256 = "b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe", @@ -732,7 +732,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "cython", sha256 = "6dcd30b5ceb887b2b965ee7ceb82ea3acb5f0642fe2206c7636b45acea4798e5", urls = [ - "http://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz", + "https://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz", "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz", ], strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17", @@ -742,7 +742,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "bazel_toolchains", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", # "https://github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", ], sha256 = "46187270ca04ff8109980f45c3438fabfe48695e163789096eb82ee097ffe685", -- GitLab From e0e4f693978dcaf5bf4ecbc18e6926bdf33b2870 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Thu, 19 Oct 2017 16:28:57 -0700 Subject: [PATCH 178/573] [tf.contrib.seq2seq] Reserve -1s in GatherTree for error states. GatherTree now emits end_token after the first decoded end_token in the path, instead of -1s at the end of each sequence. PiperOrigin-RevId: 172816652 --- .../seq2seq/kernels/beam_search_ops.cc | 7 +++-- .../seq2seq/kernels/beam_search_ops_gpu.cu.cc | 25 +++++++++------ .../contrib/seq2seq/ops/beam_search_ops.cc | 11 ++++--- .../kernel_tests/beam_search_ops_test.py | 31 ++++++++++--------- 4 files changed, 45 insertions(+), 29 deletions(-) diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc index 95273e2b33..64973ccccd 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc @@ -112,7 +112,7 @@ struct GatherTree { const int32 max_time = parent_ids.dimension(0); const int32 batch_size = parent_ids.dimension(1); const int32 beam_width = parent_ids.dimension(2); - beams.setConstant(-1); + beams.setConstant(end_token); auto DoWork = [&, ctx, end_token](int start_batch_beam, int limit_batch_beam) { @@ -138,10 +138,13 @@ struct GatherTree { beams(level, batch, beam) = step_ids(level, batch, parent); parent = parent_ids(level, batch, parent); } + // Not necessary when using a BeamSearchDecoder, but necessary + // when a user feeds in possibly broken trajectory (i.e., non-eos + // entries in a beam following eos entries). bool finished = false; for (int32 time = 0; time < max_seq_len_b; ++time) { if (finished) { - beams(time, batch, beam) = -1; + beams(time, batch, beam) = end_token; } else if (beams(time, batch, beam) == end_token) { finished = true; } diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc index e71efc48ce..bc28d492fe 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc @@ -46,24 +46,31 @@ __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time, const int32 initial_beam_ix = GET_IX(max_seq_len_b - 1, beam); beams[initial_beam_ix] = ldg(step_ids + initial_beam_ix); int32 parent = ldg(parent_ids + initial_beam_ix); + bool found_bad = false; for (int32 level = max_seq_len_b - 2; level >= 0; --level) { const int32 level_beam_ix = GET_IX(level, beam); const int32 level_parent_ix = GET_IX(level, parent); if (parent < 0 || parent > beam_width) { beams[level_beam_ix] = -1; parent = -1; + found_bad = true; } else { beams[level_beam_ix] = ldg(step_ids + level_parent_ix); parent = ldg(parent_ids + level_parent_ix); } } - bool finished = false; - for (int32 time = 0; time < max_seq_len_b; ++time) { - const int32 level_beam_ix = GET_IX(time, beam); - if (finished) { - beams[level_beam_ix] = -1; - } else if (beams[level_beam_ix] == end_token) { - finished = true; + // Not necessary when using a BeamSearchDecoder, but necessary + // when a user feeds in possibly broken trajectory (i.e., non-eos + // entries in a beam following eos entries). + if (!found_bad) { + bool finished = false; + for (int32 time = 0; time < max_seq_len_b; ++time) { + const int32 level_beam_ix = GET_IX(time, beam); + if (finished) { + beams[level_beam_ix] = end_token; + } else if (beams[level_beam_ix] == end_token) { + finished = true; + } } } #undef GET_IX @@ -80,8 +87,8 @@ struct GatherTree { const int32 max_time = parent_ids.dimension(0); const int32 batch_size = parent_ids.dimension(1); const int32 beam_width = parent_ids.dimension(2); - // First kernel launch to zero things out - beams.device(d) = beams.constant(T(-1)); + // First kernel launch to "zero" things out + beams.device(d) = beams.constant(end_token); CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * beam_width, d); // clang-format off diff --git a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc index 231504bfbb..71539b6f59 100644 --- a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc +++ b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc @@ -53,11 +53,14 @@ REGISTER_OP("GatherTree") .Doc(R"doc( Calculates the full beams from the per-step ids and parent beam ids. -This op implements the following mathematical equations: +On CPU, if an out of bound parent id is found, an error is returned. +On GPU, if an out of bound parent id is found, a -1 is stored in the +corresponding output value and the execution for that beam returns early. -```python -TODO(ebrevdo): fill in -``` +For a given beam, past the time step containing the first decoded `end_token` +all values are filled in with `end_token`. + +TODO(ebrevdo): fill in the remainder of this docstring. step_ids: `[max_time, batch_size, beam_width]`. parent_ids: `[max_time, batch_size, beam_width]`. diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py index f301314872..277c5b6ef7 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py @@ -36,24 +36,26 @@ class GatherTreeTest(test.TestCase): def testGatherTreeOne(self): # (max_time = 4, batch_size = 1, beams = 3) + end_token = 10 step_ids = _transpose_batch_time( [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]]]) max_sequence_lengths = [3] - expected_result = _transpose_batch_time( - [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]]) + expected_result = _transpose_batch_time([[[2, 2, 2], [6, 5, 6], [7, 8, 9], + [10, 10, 10]]]) beams = beam_search_ops.gather_tree( step_ids=step_ids, parent_ids=parent_ids, max_sequence_lengths=max_sequence_lengths, - end_token=10) + end_token=end_token) with self.test_session(use_gpu=True): self.assertAllEqual(expected_result, beams.eval()) def testBadParentValuesOnCPU(self): # (batch_size = 1, max_time = 4, beams = 3) # bad parent in beam 1 time 1 + end_token = 10 step_ids = _transpose_batch_time( [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( @@ -64,7 +66,7 @@ class GatherTreeTest(test.TestCase): step_ids=step_ids, parent_ids=parent_ids, max_sequence_lengths=max_sequence_lengths, - end_token=10) + end_token=end_token) with self.test_session(): with self.assertRaisesOpError( r"parent id -1 at \(batch, time, beam\) == \(0, 0, 1\)"): @@ -77,19 +79,20 @@ class GatherTreeTest(test.TestCase): return # (max_time = 4, batch_size = 1, beams = 3) # bad parent in beam 1 time 1; appears as a negative index at time 0 + end_token = 10 step_ids = _transpose_batch_time( [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) max_sequence_lengths = [3] - expected_result = _transpose_batch_time( - [[[2, -1, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]]) + expected_result = _transpose_batch_time([[[2, -1, 2], [6, 5, 6], [7, 8, 9], + [10, 10, 10]]]) with ops.device("/device:GPU:0"): beams = beam_search_ops.gather_tree( step_ids=step_ids, parent_ids=parent_ids, max_sequence_lengths=max_sequence_lengths, - end_token=10) + end_token=end_token) with self.test_session(use_gpu=True): self.assertAllEqual(expected_result, beams.eval()) @@ -115,24 +118,24 @@ class GatherTreeTest(test.TestCase): self.assertEqual((max_time, batch_size, beam_width), beams.shape) beams_value = beams.eval() for b in range(batch_size): - # Past max_sequence_lengths[b], we emit all -1s. + # Past max_sequence_lengths[b], we emit all end tokens. b_value = beams_value[max_sequence_lengths[b]:, b, :] - self.assertAllClose(b_value, -1. * np.ones_like(b_value)) + self.assertAllClose(b_value, end_token * np.ones_like(b_value)) for batch, beam in itertools.product( range(batch_size), range(beam_width)): v = np.squeeze(beams_value[:, batch, beam]) if end_token in v: + found_bad = np.where(v == -1)[0] + self.assertEqual(0, len(found_bad)) found = np.where(v == end_token)[0] - # Should be up to 1 instance of end_token per beam. - self.assertEqual(len(found), 1) - found = found[0] + found = found[0] # First occurrence of end_token. # If an end_token is found, everything before it should be a # valid id and everything after it should be -1. if found > 0: self.assertAllEqual( v[:found - 1] >= 0, np.ones_like(v[:found - 1], dtype=bool)) - self.assertAllClose( - v[found + 1:], -1 * np.ones_like(v[found + 1:])) + self.assertAllClose(v[found + 1:], + end_token * np.ones_like(v[found + 1:])) if __name__ == "__main__": -- GitLab From 2977dccc96c343ca85cb00b50672b36c99656532 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 19 Oct 2017 16:42:14 -0700 Subject: [PATCH 179/573] Context-specific C API to set options other than configproto (still unused) PiperOrigin-RevId: 172818175 --- tensorflow/c/eager/c_api.cc | 17 ++++++++++---- tensorflow/c/eager/c_api.h | 20 ++++++++++++++-- tensorflow/c/eager/c_api_internal.h | 4 ++++ tensorflow/c/eager/c_api_test.cc | 36 ++++++++++++++--------------- tensorflow/python/eager/context.py | 16 ++++++++----- tensorflow/python/pywrap_tfe.i | 16 ++++++++++++- 6 files changed, 78 insertions(+), 31 deletions(-) diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 514a4010bc..334c02bff9 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -54,9 +54,18 @@ string DeviceName(tensorflow::Device* d) { extern "C" { -TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) { +TFE_ContextOptions* TFE_NewContextOptions() { return new TFE_ContextOptions; } + +void TFE_ContextOptionsSetConfig(TFE_ContextOptions* options, const void* proto, + size_t proto_len, TF_Status* status) { + TF_SetConfig(&options->session_options, proto, proto_len, status); +} + +void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; } + +TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { TF_Graph* graph = TF_NewGraph(); - TF_Session* session = TF_NewSession(graph, opts, status); + TF_Session* session = TF_NewSession(graph, &opts->session_options, status); if (status->status.ok()) { if (session->device_mgr == nullptr || session->devices.empty()) { status->status = tensorflow::errors::InvalidArgument( @@ -72,8 +81,8 @@ TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) { TFE_Context* ret = new TFE_Context(session); ret->pflr.reset(new tensorflow::ProcessFunctionLibraryRuntime( - ret->session->device_mgr, opts->options.env, TF_GRAPH_DEF_VERSION, - &ret->func_lib_def, {})); + ret->session->device_mgr, opts->session_options.options.env, + TF_GRAPH_DEF_VERSION, &ret->func_lib_def, {})); ret->rendezvous = new tensorflow::IntraProcessRendezvous(ret->session->device_mgr); diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h index 9bfa63711b..201cb222c9 100644 --- a/tensorflow/c/eager/c_api.h +++ b/tensorflow/c/eager/c_api.h @@ -43,14 +43,30 @@ limitations under the License. extern "C" { #endif +typedef struct TFE_ContextOptions TFE_ContextOptions; + +// Return a new options object. +TF_CAPI_EXPORT extern TFE_ContextOptions* TFE_NewContextOptions(); + +// Set the config in TF_ContextOptions.options. +// config should be a serialized tensorflow.ConfigProto proto. +// If config was not parsed successfully as a ConfigProto, record the +// error information in *status. +TF_CAPI_EXPORT extern void TFE_ContextOptionsSetConfig( + TFE_ContextOptions* options, const void* proto, size_t proto_len, + TF_Status* status); + +// Destroy an options object. +TF_CAPI_EXPORT extern void TFE_DeleteContextOptions(TFE_ContextOptions*); + // "Context" under which operations/functions are executed. It encapsulates // things like the available devices, resource manager etc. // // TODO(ashankar): Merge with TF_Session? typedef struct TFE_Context TFE_Context; -TF_CAPI_EXPORT extern TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, - TF_Status* status); +TF_CAPI_EXPORT extern TFE_Context* TFE_NewContext( + const TFE_ContextOptions* opts, TF_Status* status); TF_CAPI_EXPORT extern void TFE_DeleteContext(TFE_Context* ctx, TF_Status* status); TF_CAPI_EXPORT extern TF_DeviceList* TFE_ContextListDevices(TFE_Context* ctx, TF_Status* status); diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h index 712526f170..7a440a5a7e 100644 --- a/tensorflow/c/eager/c_api_internal.h +++ b/tensorflow/c/eager/c_api_internal.h @@ -35,6 +35,10 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/thread_annotations.h" +struct TFE_ContextOptions { + TF_SessionOptions session_options; +}; + struct TFE_Context { explicit TFE_Context(TF_Session* s) : session(s) {} diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc index 72e0fe8a15..5344956ee7 100644 --- a/tensorflow/c/eager/c_api_test.cc +++ b/tensorflow/c/eager/c_api_test.cc @@ -62,10 +62,10 @@ TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b) { void BM_InitOp(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* m = TestMatrixTensorHandle(); tensorflow::testing::StartTiming(); @@ -84,10 +84,10 @@ BENCHMARK(BM_InitOp); void BM_Execute(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* m = TestMatrixTensorHandle(); TFE_Op* matmul = MatMulOp(ctx, m, m); @@ -109,9 +109,9 @@ BENCHMARK(BM_Execute); TEST(CAPI, Context) { TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TF_DeviceList* devices = TFE_ContextListDevices(ctx, status); EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); @@ -150,9 +150,9 @@ TEST(CAPI, TensorHandle) { TEST(CAPI, TensorHandleCopyBetweenDevices) { std::unique_ptr status( TF_NewStatus(), TF_DeleteStatus); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status.get()); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); TFE_TensorHandle* hcpu = TestMatrixTensorHandle(); @@ -218,10 +218,10 @@ TEST(CAPI, TensorHandleCopyBetweenDevices) { TEST(CAPI, Execute) { TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* m = TestMatrixTensorHandle(); TFE_Op* matmul = MatMulOp(ctx, m, m); @@ -285,10 +285,10 @@ string MatMulFunction() { TEST(CAPI, FunctionDefAndExecute) { TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); string function_def = MatMulFunction(); TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), @@ -326,10 +326,10 @@ TEST(CAPI, FunctionDefAndExecute) { void BM_ExecuteFunction(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); string function_def = MatMulFunction(); TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), @@ -406,10 +406,10 @@ TEST(CAPI, Variables) { // Variables use resource handles, so this is really a test for resource // tensor handling. TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* var_handle = CreateVariable(ctx, 12.0, status); ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); @@ -446,10 +446,10 @@ TEST(CAPI, Variables) { void BM_ReadVariable(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* var_handle = CreateVariable(ctx, 5.0, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index aa7cba56de..58581283d2 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -26,7 +26,6 @@ import threading from tensorflow.python import pywrap_tensorflow from tensorflow.python.framework import device as pydev from tensorflow.python.framework import errors -from tensorflow.python.util import compat from tensorflow.python.util import tf_contextlib GRAPH_MODE = 0 @@ -103,11 +102,16 @@ class Context(object): if self._context_handle is not None: return assert self._context_devices is None - opts = pywrap_tensorflow.TF_NewSessionOptions( - target=compat.as_bytes(""), config=self._config) - with errors.raise_exception_on_not_ok_status() as status: - self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status) - pywrap_tensorflow.TF_DeleteSessionOptions(opts) + opts = pywrap_tensorflow.TFE_NewContextOptions() + try: + with errors.raise_exception_on_not_ok_status() as status: + if self._config is not None: + config_str = self._config.SerializeToString() + pywrap_tensorflow.TFE_ContextOptionsSetConfig( + opts, config_str, len(config_str), status) + self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status) + finally: + pywrap_tensorflow.TFE_DeleteContextOptions(opts) # Store list of devices self._context_devices = [] with errors.raise_exception_on_not_ok_status() as status: diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 5c624a9c12..36c09c20c2 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -30,12 +30,25 @@ limitations under the License. %rename("%s") TFE_Py_TapeDeleteTrace; %rename("%s") TFE_Py_TapeRecordOperation; %rename("%s") TFE_Py_TapeExport; - +%rename("%s") TFE_NewContextOptions; +%rename("%s") TFE_ContextOptionsSetConfig; +%rename("%s") TFE_DeleteContextOptions; %{ #include "tensorflow/python/eager/pywrap_tfe.h" %} +%typemap(in) (const void* proto) { + char* c_string; + Py_ssize_t py_size; + // PyBytes_AsStringAndSize() does not copy but simply interprets the input + if (PyBytes_AsStringAndSize($input, &c_string, &py_size) == -1) { + // Python has raised an error (likely TypeError or UnicodeEncodeError). + SWIG_fail; + } + $1 = static_cast(c_string); +} + %typemap(out) TF_DataType { $result = PyInt_FromLong($1); } @@ -165,3 +178,4 @@ limitations under the License. %typemap(in, numinputs=0) TF_Status *out_status; %typemap(freearg) (TF_Status* out_status); %typemap(argout) (TFE_OutputTensorHandles* outputs, TF_Status* out_status); +%typemap(in) (const void* proto); -- GitLab From e885d1abdce5db4a67e0b3ba85dbcc708f856645 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 19 Oct 2017 16:42:45 -0700 Subject: [PATCH 180/573] One less error message in gradients_function PiperOrigin-RevId: 172818233 --- tensorflow/python/eager/backprop.py | 11 ++++------- tensorflow/python/eager/backprop_test.py | 8 ++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index da17be05b7..9580e84847 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -396,12 +396,11 @@ def implicit_grad(f): return grad_fn -def _get_arg_spec(f, params): +def _get_arg_spec(f, params, param_args): args = tf_inspect.getargspec(f).args if params is None: if not args: - raise ValueError("When params is None the differentiated function cannot" - " only take arguments by *args and **kwds.") + return range(len(param_args)) return range(len(args)) elif all(isinstance(x, six.string_types) for x in params): return [args.index(n) for n in params] @@ -560,10 +559,9 @@ def val_and_grad_function(f, params=None): ValueError: if the params are not all strings or all integers. """ - parameter_positions = _get_arg_spec(f, params) - def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" + parameter_positions = _get_arg_spec(f, params, args) dy = kwds.pop("dy", None) if dy is not None: dy = ops.convert_to_tensor(dy) @@ -616,10 +614,9 @@ def make_vjp(f, params=None): """ - parameter_positions = _get_arg_spec(f, params) - def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" + parameter_positions = _get_arg_spec(f, params, args) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() sources = [] diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 95d5f0adcb..7da8eb0c9b 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -381,6 +381,14 @@ class BackpropTest(test.TestCase): [tensor_shape.TensorShape(s).as_proto() for s in shape_list], backprop.make_attr([pywrap_tensorflow.TF_ATTR_SHAPE], shape_list)) + def testArgsGradientFunction(self): + + def f(*args): + return args[0] * args[0] + + grad = backprop.gradients_function(f) + self.assertAllEqual(grad(1.0)[0], 2.0) + def testMultiValueConvertToTensor(self): x = resource_variable_ops.ResourceVariable( initial_value=array_ops.constant([1.0]), name='x') -- GitLab From f1054553eafc74df8be9425c3344e71af98962ad Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 17:31:49 -0700 Subject: [PATCH 181/573] Add missing backslash in macro in mkl_transpose_op.cc. Fix erroneous formatting that resulted from it. Fix return type for function template MKLTranspose2D. Define MKL_Complex8 and MKL_Complex16 macros before including the MKL headers. Only conjugate but don't transpose if conjugate=true && perm[0] == 0 && perm[1] == 1. PiperOrigin-RevId: 172824073 --- tensorflow/core/kernels/mkl_transpose_op.cc | 120 ++++++++++---------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/tensorflow/core/kernels/mkl_transpose_op.cc b/tensorflow/core/kernels/mkl_transpose_op.cc index 89a1d5e8a7..764d4c9400 100644 --- a/tensorflow/core/kernels/mkl_transpose_op.cc +++ b/tensorflow/core/kernels/mkl_transpose_op.cc @@ -18,6 +18,9 @@ limitations under the License. #ifdef INTEL_MKL #define EIGEN_USE_THREADS +#include "tensorflow/core/framework/numeric_types.h" +#define MKL_Complex8 tensorflow::complex64 +#define MKL_Complex16 tensorflow::complex128 #include "mkl_trans.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/kernels/transpose_op.h" @@ -41,7 +44,7 @@ namespace tensorflow { namespace { template -void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {} +Status MKLTranspose2D(const char trans, const Tensor& in, Tensor* out); // Documentation here: https://software.intel.com/en-us/node/520863 // Parameters: (ordering:row-major, operation:transpose, num_rows, num_cols, @@ -54,70 +57,73 @@ void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {} mkl_##PREFIX##omatcopy('R', trans, in.dim_size(0), in.dim_size(1), 1, \ in.flat().data(), in.dim_size(1), \ out->flat().data(), in.dim_size(0)); \ - return Status::OK(); + return Status::OK(); \ } - INSTANTIATE(float, s) - INSTANTIATE(double, d) - INSTANTIATE(complex64, c) - INSTANTIATE(complex128, z) +INSTANTIATE(float, s) +INSTANTIATE(double, d) +INSTANTIATE(complex64, c) +INSTANTIATE(complex128, z) #undef INSTANTIATE - static const char kMKLTranspose = 'T'; - static const char kMKLConjugateTranspose = 'C'; - - } // namespace tensorflow - - Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in, - gtl::ArraySlice perm, - Tensor* out) { - if (in.dims() == 2) { - switch (in.dtype()) { - case DT_FLOAT: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_DOUBLE: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_COMPLEX64: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_COMPLEX128: - return MKLTranspose2D(kMKLTranspose, in, out); - default: - break; - } +static const char kMKLTranspose = 'T'; +static const char kMKLConjugateTranspose = 'C'; + +} // namespace + +Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in, + gtl::ArraySlice perm, + Tensor* out) { + if (in.dims() == 2) { + if (perm[0] == 0 && perm[1] == 1) { + return Status::OK(); + } + switch (in.dtype()) { + case DT_FLOAT: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_DOUBLE: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_COMPLEX64: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_COMPLEX128: + return MKLTranspose2D(kMKLTranspose, in, out); + default: + break; } - // Fallback to eigen if transpose parameters not supported by MKL - typedef Eigen::ThreadPoolDevice CPUDevice; - return ::tensorflow::DoTranspose(ctx->eigen_device(), in, perm, - out); } - - Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx, - const Tensor& in, - gtl::ArraySlice perm, - Tensor* out) { - if (in.dims() == 2) { - // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels - // for any transpose that can be reduced to swapping the last two - // dimensions in a rank-3 tensor. We can even run each outer dimension in - // a separate thread. - switch (in.dtype()) { - case DT_FLOAT: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_DOUBLE: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_COMPLEX64: - return MKLTranspose2D(kMKLConjugateTranspose, in, out); - case DT_COMPLEX128: - return MKLTranspose2D(kMKLConjugateTranspose, in, out); - default: - break; - } + // Fallback to eigen if transpose parameters not supported by MKL + typedef Eigen::ThreadPoolDevice CPUDevice; + return ::tensorflow::DoTranspose(ctx->eigen_device(), in, perm, + out); +} + +Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx, + const Tensor& in, + gtl::ArraySlice perm, + Tensor* out) { + if (in.dims() == 2 && perm[0] == 1 && perm[1] == 0) { + // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels + // for any transpose that can be reduced to swapping the last two + // dimensions in a rank-3 tensor. We can even run each outer dimension in + // a separate thread. + switch (in.dtype()) { + case DT_FLOAT: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_DOUBLE: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_COMPLEX64: + return MKLTranspose2D(kMKLConjugateTranspose, in, out); + case DT_COMPLEX128: + return MKLTranspose2D(kMKLConjugateTranspose, in, out); + default: + break; } - // Fallback to eigen if transpose parameters not supported by MKL - typedef Eigen::ThreadPoolDevice CPUDevice; - return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), - in, perm, out); } + // Fallback to eigen if transpose parameters not supported by MKL + typedef Eigen::ThreadPoolDevice CPUDevice; + return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), in, + perm, out); +} } // namespace tensorflow -- GitLab From e7654b99c46a479d61c1fd96a9f4710682acf4da Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Thu, 19 Oct 2017 17:44:32 -0700 Subject: [PATCH 182/573] Adds tfe.IsolateTest, an Eager-agnostic abstraction for isolating resources Switches Eager unit tests to use IsolateTest, so their resources have unique container names. PiperOrigin-RevId: 172825317 --- tensorflow/contrib/eager/python/tfe.py | 2 + tensorflow/python/eager/graph_callable.py | 10 +++ tensorflow/python/framework/test_util.py | 64 ++++++++++++++++- tensorflow/python/framework/test_util_test.py | 71 +++++++++++++++++++ .../resource_variable_ops_test.py | 11 ++- .../python/ops/resource_variable_ops.py | 11 +++ 6 files changed, 164 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 25942aadfb..4ed258f6ff 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -53,6 +53,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@in_eager_mode @@in_graph_mode +@@IsolateTest @@run_test_in_graph_and_eager_modes """ @@ -84,6 +85,7 @@ from tensorflow.python.eager.execution_callbacks import nan_callback from tensorflow.python.eager.execution_callbacks import seterr from tensorflow.python.framework.ops import enable_eager_execution from tensorflow.python.framework.ops import eager_run as run +from tensorflow.python.framework.test_util import IsolateTest from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable from tensorflow.python.util.all_util import remove_undocumented diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index 3aba164630..0ec83636a0 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -312,11 +312,21 @@ def _graph_callable_internal(func, shape_and_dtypes): Returns: Callable graph object. """ + container = tf_ops.get_default_graph()._container # pylint: disable=protected-access + container_prefix = tf_ops.get_default_graph()._container_prefix # pylint: disable=protected-access with context.graph_mode(): # This graph will store both the initialization and the call version of the # wrapped function. It will later be used by the backprop code to build the # backprop graph, if necessary. tmp_graph = tf_ops.Graph() + # Inherit the container from the original graph to create resources at user + # expected containers. Also inherits the container prefix, since this is + # used for error checking when isolating Eager execution (the container + # prefix at creation must match the container prefix when used, and + # variables returned from the graph callable will be used in the outside + # context). + tmp_graph._container = container # pylint: disable=protected-access + tmp_graph._container_prefix = container_prefix # pylint: disable=protected-access with tmp_graph.as_default(): # Placeholders for the non-variable inputs. func_inputs = _get_graph_callable_inputs(shape_and_dtypes) diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index c681ffb514..a01bf02deb 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -47,6 +47,7 @@ from tensorflow.python import pywrap_tensorflow from tensorflow.python.client import device_lib from tensorflow.python.client import session from tensorflow.python.eager import context +from tensorflow.python.eager import tape from tensorflow.python.framework import device as pydev from tensorflow.python.framework import errors from tensorflow.python.framework import ops @@ -391,6 +392,66 @@ def with_c_api(cls): return cls +class IsolateTest(object): + """A context manager which isolates resources in its block. + + Provides an Eager-agnostic abstraction for preventing the sharing of + variables and other resources. + + In graph mode, resource handle ops are only executed in a particular Session, + isolating them from resources with the same name in other Graphs. In Eager, + separate Sessions do not exist, so resources (particularly ResourceVariables) + would be shared implicitly if a resource of the same name were created + anywhere in a Python process. Multiple handles to the same resource would + cause several issues, and so this type of sharing will raise an exception. + + Using resources with the same name in a single Python process may be useful + (especially for unit tests), so this context manager provides an abstraction + for isolating resources. Using a resource created in one Isolation environment + in another is an error. + + Example usage in Eager mode: + + ```python + import tensorflow as tf + # Import subject to change + from tensorflow.contrib.eager.python import tfe + + tfe.enable_eager_execution() + + for hyperparameter in [1, 2, 3]: + with tfe.IsolateTest(): + v = tfe.Variable(name="v", initial_value=hyperparameter) + # train model, test results ... + ``` + + IsolateTest is currently exposed through contrib.eager, but it creates a new + default Graph and provides equivalent safety in graph mode. + """ + + def __init__(self): + if context.in_eager_mode() and tape.could_possibly_record(): + raise ValueError("Cannot isolate Eager execution with an active tape.") + # In Eager, Graphs set a container which isolates resources, and maintain a + # VariableStore which caches ResourceVariable objects created through + # get_variable. So setting the default Graph has the side effect of + # isolating Eager resources. + with context.eager_mode(): + # Create the graph in Eager mode, as this provides stricter semantics + # (i.e. has a unique container prefix). This prevents implicit sharing + # when a Graph-mode graph is created and then Eager mode is enabled (an + # error through enable_eager_execution, but common with context managers + # in unit tests). + self._graph_as_default_context_manager = ops.Graph().as_default() + + def __enter__(self): + self._graph_as_default_context_manager.__enter__() + + def __exit__(self, type_arg, value_arg, traceback_arg): + return self._graph_as_default_context_manager.__exit__( + type_arg, value_arg, traceback_arg) + + def run_in_graph_and_eager_modes(__unused__=None, graph=None, config=None, use_gpu=False, force_gpu=False, reset_test=True): @@ -440,9 +501,8 @@ def run_in_graph_and_eager_modes(__unused__=None, graph=None, config=None, with context.device("/device:CPU:0"): f(self, **kwargs) - eager_graph = graph or ops.Graph() with context.eager_mode(): - with eager_graph.as_default(): + with IsolateTest(): run_eager_mode() return decorated diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py index 6129fa2e0d..b2f8d62095 100644 --- a/tensorflow/python/framework/test_util_test.py +++ b/tensorflow/python/framework/test_util_test.py @@ -27,12 +27,16 @@ from google.protobuf import text_format from tensorflow.core.framework import graph_pb2 from tensorflow.core.protobuf import meta_graph_pb2 +from tensorflow.python.client import session +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables from tensorflow.python.platform import googletest @@ -325,5 +329,72 @@ class TestUtilTest(test_util.TensorFlowTestCase): self.assertEqual(a_rand, b_rand) +@test_util.with_c_api +class IsolationTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes() + def test_variable_reuse_exception(self): + with test_util.IsolateTest(), session.Session(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="first_container_variable", + initial_value=1) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + with test_util.IsolateTest(): + if context.in_graph_mode(): + with self.assertRaises(RuntimeError): + self.evaluate(first_container_variable.read_value()) + else: + with self.assertRaises(ValueError): + first_container_variable.read_value() + + @test_util.run_in_graph_and_eager_modes() + def test_variable_reuse_exception_nested(self): + with test_util.IsolateTest(), session.Session(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="first_container_variable", + initial_value=1) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + with test_util.IsolateTest(), session.Session(): + if context.in_graph_mode(): + with self.assertRaises(RuntimeError): + self.evaluate(first_container_variable.read_value()) + else: + with self.assertRaises(ValueError): + first_container_variable.read_value() + + @test_util.run_in_graph_and_eager_modes() + def test_no_sharing(self): + with test_util.IsolateTest(), session.Session(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="same_name", + initial_value=1) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + with test_util.IsolateTest(), session.Session(): + second_container_variable = resource_variable_ops.ResourceVariable( + name="same_name", + initial_value=2) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + self.assertEqual( + 2, self.evaluate(second_container_variable.read_value())) + self.assertEqual(1, self.evaluate(first_container_variable.read_value())) + + def test_graph_mode_isolation(self): + with context.graph_mode(): + # Even if we've (accidentally) called IsolateTest in Graph mode, it should + # provide Eager isolation. + with test_util.IsolateTest(): + with context.eager_mode(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="first_container_variable", + initial_value=1) + with context.eager_mode(): + with self.assertRaises(ValueError): + first_container_variable.read_value() + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 23676223dc..cf4b61674f 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -309,12 +309,15 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.evaluate(variables.global_variables_initializer()) w = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4") + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4", + # Needed in Eager since we get a unique container name by default. + container=ops.get_default_graph()._container) w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) self.assertEqual(300.0, self.evaluate(w_read)) x = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5") + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5", + container=ops.get_default_graph()._container) with self.assertRaisesOpError("Resource .*/var5/.* does not exist"): x_read = resource_variable_ops.read_variable_op(x, v.dtype.base_dtype) self.evaluate(x_read) @@ -328,7 +331,9 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.evaluate(variables.global_variables_initializer()) w = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6") + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6", + # Needed in Eager since we get a unique container name by default. + container=ops.get_default_graph()._container) w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) self.assertEqual(300.0, self.evaluate(w_read)) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index dd3f167145..aa45752a9d 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -270,6 +270,9 @@ class ResourceVariable(variables.Variable): collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES] self._save_slice_info = None self._in_graph_mode = context.in_graph_mode() + # Save the graph's container prefix for error checking. Reading the value of + # the ResourceVariable from another Graph in Eager mode is an error. + self._container_prefix = ops.get_default_graph()._container_prefix # pylint: disable=protected-access with ops.control_dependencies(None): with ops.name_scope(name, "Variable", [] if init_from_fn else [initial_value]) as name: @@ -577,7 +580,15 @@ class ResourceVariable(variables.Variable): Returns: the read operation. + Raises: + ValueError: if the ResourceVariable was created in another isolation + environment or graph. """ + if (not self._in_graph_mode and + self._container_prefix != ops.get_default_graph()._container_prefix): # pylint: disable=protected-access + raise ValueError( + "Attempted to read a variable from another isolation environment" + " or Graph") with ops.name_scope("Read"): # Ensure we read the variable in the same device as the handle. with ops.device(self._handle_device): -- GitLab From db07ee27b75f5efecf3f3706ec1a11e4cd05da54 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 19 Oct 2017 17:58:38 -0700 Subject: [PATCH 183/573] Fix bug introduced in https://github.com/tensorflow/tensorflow/commit/dc442f4ce2d3b11b56721337fe2b9e2282be93be Potentially invalid pointers passed to GraphConstructor::Construct() PiperOrigin-RevId: 172826567 --- tensorflow/core/graph/graph_constructor.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index 92b4843221..b2c193b050 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -1068,10 +1068,16 @@ Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef, refiner->set_graph_def_version( std::min(refiner->graph_def_version(), gdef.versions().producer())); - return GraphConstructor::Construct( - opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner, - &results->return_tensors, &results->return_nodes, - &results->unused_input_map_keys); + if (results == nullptr) { + return GraphConstructor::Construct(opts, gdef.node(), &gdef.versions(), + &gdef.library(), g, refiner, nullptr, + nullptr, nullptr); + } else { + return GraphConstructor::Construct( + opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner, + &results->return_tensors, &results->return_nodes, + &results->unused_input_map_keys); + } } void CopyGraph(const Graph& src, Graph* dest) { -- GitLab From fa4d04ab99d45eb317e39c1a6b8848bbc47ebe0e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 17:58:50 -0700 Subject: [PATCH 184/573] Address Metrics TODOs, in particular we'd like them to work in Graph mode. To get this to work, we add support for capturing tensors from outside the function graph in graph mode to eager/function.py. Also get unique names and variable scopes working. PiperOrigin-RevId: 172826589 --- tensorflow/contrib/eager/python/BUILD | 9 +- .../contrib/eager/python/evaluator_test.py | 2 +- .../contrib/eager/python/metrics_impl.py | 156 ++++++++++++------ .../contrib/eager/python/metrics_test.py | 51 ++++++ tensorflow/python/eager/function.py | 61 +++++-- tensorflow/python/eager/function_test.py | 18 ++ tensorflow/tools/ci_build/ci_sanity.sh | 1 + 7 files changed, 234 insertions(+), 64 deletions(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 0c61630aa8..702136e3e4 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -132,11 +132,12 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", "//tensorflow/python:layers_base", "//tensorflow/python:math_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:function", ], ) @@ -146,6 +147,10 @@ py_test( srcs_version = "PY2AND3", deps = [ ":metrics", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:variables", + "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], ) @@ -160,6 +165,8 @@ py_library( deps = [ ":datasets", ":metrics", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:function", ], ) diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py index 099e10e230..b18463c31a 100644 --- a/tensorflow/contrib/eager/python/evaluator_test.py +++ b/tensorflow/contrib/eager/python/evaluator_test.py @@ -86,7 +86,7 @@ class EvaluatorTest(test.TestCase): for v in e.metric_variables: p = v.name.split("/")[0] prefix_count[p] = prefix_count.get(p, 0) + 1 - self.assertEqual({"outer-mean": 2, "mean": 2}, prefix_count) + self.assertEqual({"outer_mean": 2, "mean": 2}, prefix_count) def testDataset(self): e = SimpleEvaluator(IdentityModel()) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 63a0f8d9a4..2a624b218c 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -18,6 +18,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import re + +from tensorflow.python.eager import context +from tensorflow.python.eager import function from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -25,55 +29,69 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope +_to_replace = re.compile("[^A-Za-z0-9.]") + + class Metric(object): """A metric holds state for aggregating statistics over an evaluation run. Users will use Evaluator.add_metric() to add Metric objects to their - evaluation, call them in each step, and then use - Evaluator.all_metric_results() at the end. + evaluation, call them in each step (treating the object as a callable), + and then use Evaluator.all_metric_results() at the end. Descendants will implement: - * call(): Should follow this pattern: - if not self.built: - self.var = self.add_variable(...) - self.add_update(self.var.assign_add(...)) - * aggregate(): Adds in the state from a list of metrics of the same type - as `self`. (Default of summing all the variables will be fine for most - descendants.) - * result(): Computes and returns a final value for the metric + * `build()`: All variables should be created in this method, by calling + `self.add_variable()` as in: `self.var = self.add_variable(...)` + build() will be called in the first invocation of `__call__()`, with + the same arguments passed `call()`. + * `call()`: Has all updates to variables, as in: + self.var.assign_add(...) + * `result()`: Computes and returns a final value for the metric from the variables in `self`. + + Decendants may override, but usually won't need to: + * `aggregate()`: Adds in the state from a list of metrics of the same type + as `self`. (Default is to sum all the variables.) + * `reset()`: Reset all variables to their initial state. (Default is to + zero all the variables.) + Note that users should not call `aggregate()` or `reset()`, they are for + use by TensorFlow infrastructure. """ def __init__(self, name=None): - self.built = False + self._built = False self._vars = [] self._updates = [] - self._name = name or self.__class__.__name__ - # TODO(josh11b): Need some way to make sure two Metrics in the same - # Network have distinct names. Maybe we can get a unique name from - # a name/variable scope? - # TODO(josh11b): self._in_graph_mode = context.in_graph_mode() + name = name or self.__class__.__name__ + # Replace things like spaces in name to create a valid scope name. + scope_name = _to_replace.sub("_", name) + # We create the variable scope now to get the unique name that will + # be used as a variable prefix when build() calls add_variable(). + with variable_scope.variable_scope( + None, default_name=scope_name, use_resource=True, reuse=False) as scope: + pos = scope.name.rfind(scope_name) + self._name = name + scope.name[pos + len(scope_name):] + self._scope = scope + if context.in_graph_mode(): + # We make self.call() into a graph callable here, so that we can + # return a single op that performs all of the variable updates. + self.call = function.defun(self.call) # ---- API for users ---- def __call__(self, *args, **kwargs): - # TODO(josh11b): If self._in_graph_mode is true, make self.call() into a - # graph callable here, so that variable updates happen without requiring - # a separate fetch. - # TODO(josh11b): Do we need a separate build() method to separate - # initialization from each update? If so, how do we get the arguments - # to it? We *could* just pass in *args and **kwargs... - if not self.built: - # TODO(ashankar): Set up container isolation so there is no chance - # distinct metrics objects accidentally share variables. - # TODO(josh11b): Replace things like spaces in self._name to create - # a valid scope name. - with variable_scope.variable_scope( - self._name, use_resource=True, reuse=False): - ret = self.call(*args, **kwargs) - self.built = True - else: - ret = self.call(*args, **kwargs) - return ret + """Returns op to execute to update this metric for these inputs. + + Returns None if eager execution is enabled. + + Args: + *args: + **kwargs: A mini-batch of inputs to the Metric, passed on to `call()`. + """ + if not self._built: + with variable_scope.variable_scope(self._scope): + self.build(*args, **kwargs) + self._built = True + return self.call(*args, **kwargs) @property def name(self): @@ -84,10 +102,43 @@ class Metric(object): return self._vars # ---- To be implemented by descendants --- + def build(self, *args, **kwargs): + """Method to create variables. + + Called by `__call__()` before `call()` for the first time. + + Args: + *args: + **kwargs: The arguments to the first invocation of `__call__()`. + `build()` may use the shape and/or dtype of these arguments + when deciding how to create variables. + """ + raise NotImplementedError("Metrics must define a build() member function") + def call(self, *args, **kwargs): - """Accumulates statistics for the metric.""" + """Accumulates statistics for the metric. Users should use __call__ instead. + + Note: This function is executed as a graph function in graph mode. + This means: + a) Operations on the same resource are executed in textual order. + This should make it easier to do things like add the updated + value of a variable to another, for example. + b) You don't need to worry about collecting the update ops to execute. + All update ops added to the graph by this function will be executed. + As a result, code should generally work the same way with graph or + eager execution. + + Args: + *args: + **kwargs: A mini-batch of inputs to the Metric, as passed to + `__call__()`. + """ raise NotImplementedError("Metrics must define a call() member function") + def result(self): # TODO(josh11b): Add an optional summary_writer parameter. + """Computes and returns a final value for the metric.""" + raise NotImplementedError("Metrics must define a result() member function") + # We can support two different strategies of for doing data-parallel # distributed metric computations: # * Put metric variables on the first device and rely on small @@ -123,16 +174,19 @@ class Metric(object): self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics])) # pylint: enable=protected-access - def result(self): # TODO(josh11b): Add an optional summary_writer parameter. - """Computes and returns a final value for the metric.""" - raise NotImplementedError("Metrics must define a result() member function") + def reset(self): + """Reset this metric to a freshly initialized state. + + Default implementation zeros all the metric variables. + """ + for v in self._vars: + v.assign(math_ops.zeros_like(v)) # ---- For use by descendants --- def add_variable(self, name, shape=None, dtype=None, initializer=None): """***Only for use by descendants of Metric***.""" - if self.built: - raise RuntimeError("Can't call add_variable() after a Metric has been " - "built in the first call().") + if self._built: + raise RuntimeError("Can't call add_variable() except in build().") v = variable_scope.get_variable(name, shape, dtype, initializer, trainable=False, use_resource=True) self._vars.append(v) @@ -144,6 +198,15 @@ class Mean(Metric): # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64? # Or defaults to type of the input if it is tf.float32, else tf.float64? + def build(self, values, weights=None): + del values, weights # build() does not use call's arguments + self.numer = self.add_variable(name="numer", shape=(), + dtype=dtypes.float64, + initializer=init_ops.zeros_initializer) + self.denom = self.add_variable(name="denom", shape=(), + dtype=dtypes.float64, + initializer=init_ops.zeros_initializer) + def call(self, values, weights=None): """Accumulate statistics for computing the mean. @@ -154,13 +217,6 @@ class Mean(Metric): values: Tensor with the per-example value. weights: Optional weighting of each example. Defaults to 1. """ - if not self.built: # False only in the first call(). - self.numer = self.add_variable(name="numer", shape=(), - dtype=dtypes.float64, - initializer=init_ops.zeros_initializer) - self.denom = self.add_variable(name="denom", shape=(), - dtype=dtypes.float64, - initializer=init_ops.zeros_initializer) if weights is None: self.denom.assign_add( math_ops.cast(array_ops.size(values), dtypes.float64)) @@ -179,6 +235,10 @@ class Mean(Metric): class Accuracy(Mean): """Calculates how often `predictions` matches `labels`.""" + def build(self, labels, predictions, weights=None): + del labels, predictions, weights + super(Accuracy, self).build(None) # Arguments are unused + def call(self, labels, predictions, weights=None): """Accumulate accuracy statistics. diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 089bad5a0e..bfb79cd72e 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -19,7 +19,11 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.eager.python import metrics +from tensorflow.python.eager import context from tensorflow.python.eager import test +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variables class MetricsTest(test.TestCase): @@ -56,6 +60,53 @@ class MetricsTest(test.TestCase): m([7], [2]) # 0 correct, weight 1 self.assertEqual(2.5/5, m.result().numpy()) + def testTwoMeans(self): + # Verify two metrics with the same class and name don't + # accidentally share state. + m1 = metrics.Mean() + m2 = metrics.Mean() + m1(0) + m2(2) + self.assertEqual(0, m1.result().numpy()) + self.assertEqual(2, m2.result().numpy()) + self.assertNotEqual(m1.name, m2.name) + + def testNamesWithSpaces(self): + # Verify two metrics with the same class and name don't + # accidentally share state. + m1 = metrics.Mean("has space") + m2 = metrics.Mean("has space") + m2(2) + m1(0) + self.assertEqual(m1.name, "has space") + self.assertEqual(m1.numer.name, "has_space/numer:0") + self.assertEqual(m2.name, "has space_1") + self.assertEqual(m2.numer.name, "has_space_1/numer:0") + + def testGraph(self): + with context.graph_mode(), self.test_session() as sess: + m = metrics.Mean() + p = array_ops.placeholder(dtypes.float32) + accumulate = m(p) + variables.global_variables_initializer().run() + sess.run(accumulate, feed_dict={p: [1, 10, 100]}) + sess.run(accumulate, feed_dict={p: 1000}) + sess.run(accumulate, feed_dict={p: [10000, 100000]}) + self.assertAllEqual(m.result().eval(), 111111.0/6) + + def testTwoMeansGraph(self): + # Verify two metrics with the same class and name don't + # accidentally share state. + with context.graph_mode(), self.test_session() as sess: + m1 = metrics.Mean() + m2 = metrics.Mean() + accumulate1 = m1(0) + accumulate2 = m2(2) + variables.global_variables_initializer().run() + sess.run([accumulate1, accumulate2]) + self.assertEqual(0, m1.result().eval()) + self.assertEqual(2, m2.result().eval()) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index da49517cf9..e675ee8988 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -79,6 +79,22 @@ def capture_tensors(captures): _scoped_captures.tensors = old +def capture_value(tensor_map, value, dtype, name): + """Capture a value from outside the function, to pass in as an extra arg.""" + captured_value = tensor_map.get(ops.tensor_id(value), None) + if captured_value is None: + captured_value = graph_placeholder( + dtype=dtype or value.dtype, shape=value.shape, name=name) + if captured_value.dtype == dtypes.resource: + captured_value._handle_data = value._handle_data # pylint: disable=protected-access + tensor_map[ops.tensor_id(value)] = (value, captured_value) + else: + captured_value = captured_value[1] + tape.record_operation("captured_value", [captured_value], [value], + lambda x: [x]) + return captured_value + + def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False): """Captures a Tensor while building a graph mode function. @@ -100,18 +116,33 @@ def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False): if tensor_map is None: # Capturing is not enabled. return constant_op.constant(value.numpy()) - captured_value = tensor_map.get(ops.tensor_id(value), None) - if captured_value is None: - captured_value = graph_placeholder( - dtype=dtype or value.dtype, shape=value.shape, name=name) - if captured_value.dtype == dtypes.resource: - captured_value._handle_data = value._handle_data # pylint: disable=protected-access - tensor_map[ops.tensor_id(value)] = (value, captured_value) - else: - captured_value = captured_value[1] - tape.record_operation("captured_value", [captured_value], [value], - lambda x: [x]) - return captured_value + return capture_value(tensor_map, value, dtype, name) + + +class CapturingGraph(ops.Graph): + + def __init__(self, captures): + super(CapturingGraph, self).__init__() + self._building_function = True + self.captures = captures + + def create_op( + self, + op_type, + inputs, + dtypes, # pylint: disable=redefined-outer-name + input_types=None, + name=None, + attrs=None, + op_def=None, + compute_shapes=True, + compute_device=True): + for i, inp in enumerate(inputs): + if inp.graph is not self: + inputs[i] = capture_value(self.captures, inp, inp.dtype, inp.op.name) + return super(CapturingGraph, self).create_op( + op_type, inputs, dtypes, input_types, name, attrs, op_def, + compute_shapes, compute_device) # TODO(apassos): it'd be really nice if we could scope this registration. @@ -325,6 +356,8 @@ class _GraphModeFunction(object): name="FunctionCall", compute_shapes=False) result = op.outputs + if not result: + return op for i, s in enumerate(self._output_shapes): result[i].set_shape(s) else: @@ -381,7 +414,8 @@ def _get_defun_inputs(args): def _defun_internal(name, func, args, kwds): """Defines and returns graph-mode version of func.""" with context.graph_mode(): - tmp_graph = ops.Graph() + captures = {} + tmp_graph = CapturingGraph(captures) # Copy the graph collections to ensure summaries and other things work. This # lets the function access (but not mutate) collections of the containing # graph, such as the global step and the summary writer collections. @@ -392,7 +426,6 @@ def _defun_internal(name, func, args, kwds): with tmp_graph.as_default(): func_inputs = _get_defun_inputs(args) - captures = {} with capture_tensors(captures): func_outputs = func(*func_inputs, **kwds) ids = list(sorted(captures.keys())) diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index fb647f5c21..a4c351e8c9 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -32,6 +32,7 @@ from tensorflow.python.ops import clip_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables class FunctionTest(test.TestCase): @@ -68,6 +69,23 @@ class FunctionTest(test.TestCase): self.assertAllEqual(step(), 2.0) + def testGraphModeCaptureVariable(self): + with context.graph_mode(), self.test_session() as sess: + + class HasAVar(object): + + def __init__(self): + self.v = resource_variable_ops.ResourceVariable(1.0) + + def call(self): + return self.v * 2 + + o = HasAVar() + variables.global_variables_initializer().run() + call = function.defun(o.call) + op = call() + self.assertAllEqual(sess.run(op), 2.0) + def testTensorConversionWithDefun(self): @function.defun diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 4e72d025a2..1703cae1e5 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -95,6 +95,7 @@ do_pylint() { "^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\ "^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\ "^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\ +"^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\ "^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\ "^tensorflow/python/keras/_impl/keras/callbacks\.py.*\[E1133.*not-an-iterable" -- GitLab From 3715cffc6e2338cf2fc6ad6aba5c1d00ce598bfd Mon Sep 17 00:00:00 2001 From: Anna R Date: Thu, 19 Oct 2017 18:27:01 -0700 Subject: [PATCH 185/573] Internal change. PiperOrigin-RevId: 172829126 --- tensorflow/core/framework/api_def.proto | 5 +- tensorflow/core/framework/op_gen_lib.cc | 18 +++++++ tensorflow/core/framework/op_gen_lib_test.cc | 50 ++++++++++++++++++-- 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/framework/api_def.proto b/tensorflow/core/framework/api_def.proto index 987caee250..98c38efc0e 100644 --- a/tensorflow/core/framework/api_def.proto +++ b/tensorflow/core/framework/api_def.proto @@ -51,7 +51,8 @@ message ApiDef { // endpoints are deprecated). message Endpoint { // Name should be either like "CamelCaseName" or - // "Package.CamelCaseName". + // "Package.CamelCaseName". Client-language-specific ApiDefs may + // use a snake_case convention instead of CamelCase. string name = 1; // First GraphDef version at which the op is disallowed. @@ -74,7 +75,7 @@ message ApiDef { } repeated Arg in_arg = 4; repeated Arg out_arg = 5; - // List of post-rename in_arg names to specify new argument order. + // List of original in_arg names to specify new argument order. // Length of arg_order should be either empty to keep current order // or match size of in_arg. repeated string arg_order = 11; diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc index cfaca897ba..1e93e9be09 100644 --- a/tensorflow/core/framework/op_gen_lib.cc +++ b/tensorflow/core/framework/op_gen_lib.cc @@ -412,6 +412,8 @@ void InitApiDefFromOpDef(const OpDef& op_def, ApiDef* api_def) { api_in_arg->set_name(op_in_arg.name()); api_in_arg->set_rename_to(op_in_arg.name()); api_in_arg->set_description(op_in_arg.description()); + + *api_def->add_arg_order() = op_in_arg.name(); } for (const auto& op_out_arg : op_def.output_arg()) { auto* api_out_arg = api_def->add_out_arg(); @@ -503,6 +505,22 @@ Status MergeApiDefs(ApiDef* base_api_def, const ApiDef& new_api_def) { } // Merge arg order if (new_api_def.arg_order_size() > 0) { + // Validate that new arg_order is correct. + if (new_api_def.arg_order_size() != base_api_def->arg_order_size()) { + return errors::FailedPrecondition( + "Invalid number of arguments ", new_api_def.arg_order_size(), " for ", + base_api_def->graph_op_name(), + ". Expected: ", base_api_def->arg_order_size()); + } + if (!std::is_permutation(new_api_def.arg_order().begin(), + new_api_def.arg_order().end(), + base_api_def->arg_order().begin())) { + return errors::FailedPrecondition( + "Invalid arg_order: ", str_util::Join(new_api_def.arg_order(), ", "), + " for ", base_api_def->graph_op_name(), + ". All elements in arg_order override must match base arg_order: ", + str_util::Join(base_api_def->arg_order(), ", ")); + } base_api_def->clear_arg_order(); std::copy( new_api_def.arg_order().begin(), new_api_def.arg_order().end(), diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc index b7ee6db991..da9b4dfbb1 100644 --- a/tensorflow/core/framework/op_gen_lib_test.cc +++ b/tensorflow/core/framework/op_gen_lib_test.cc @@ -207,6 +207,8 @@ attr { name: "attr_a" rename_to: "attr_a" } +arg_order: "arg_a" +arg_order: "arg_b" )"; OpList op_list; protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT @@ -331,8 +333,8 @@ op { name: "arg_c" rename_to: "arg_cc" } - arg_order: "arg_aa" arg_order: "arg_b" + arg_order: "arg_a" } )"; OpList op_list; @@ -351,8 +353,8 @@ op { EXPECT_EQ("arg_cc", api_def->out_arg(0).rename_to()); ASSERT_EQ(2, api_def->arg_order_size()); - EXPECT_EQ("arg_aa", api_def->arg_order(0)); - EXPECT_EQ("arg_b", api_def->arg_order(1)); + EXPECT_EQ("arg_b", api_def->arg_order(0)); + EXPECT_EQ("arg_a", api_def->arg_order(1)); } TEST(OpGenLibTest, ApiDefOverrideDescriptions) { @@ -411,5 +413,47 @@ op { auto status = api_map.LoadApiDef(api_def1); ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); } + +TEST(OpGenLibTest, ApiDefInvalidArgOrder) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + arg_order: "arg_a" + arg_order: "unexpected_arg" +} +)"; + + const string api_def2 = R"( +op { + graph_op_name: "testop" + arg_order: "arg_a" +} +)"; + + const string api_def3 = R"( +op { + graph_op_name: "testop" + arg_order: "arg_a" + arg_order: "arg_a" +} +)"; + + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + + // Loading with incorrect arg name in arg_order should fail. + auto status = api_map.LoadApiDef(api_def1); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); + + // Loading with incorrect number of args in arg_order should fail. + status = api_map.LoadApiDef(api_def2); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); + + // Loading with the same argument twice in arg_order should fail. + status = api_map.LoadApiDef(api_def3); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); +} } // namespace } // namespace tensorflow -- GitLab From 0671c0b2546dbea87e231d336d5f4c0573a01964 Mon Sep 17 00:00:00 2001 From: David Soergel Date: Thu, 19 Oct 2017 19:01:19 -0700 Subject: [PATCH 186/573] Usability improvements regarding export signature generation. * Log report of which signatures are produced and which TF Serving APIs are targeted. * Improve docstrings for signature_def builders, explaining the TF Serving API constraints. * Accept a single Tensor as a prediction output (which will be named 'output'). PiperOrigin-RevId: 172831366 --- tensorflow/python/estimator/export/export.py | 56 +++++++++++++++++-- .../python/estimator/export/export_output.py | 10 ++-- .../estimator/export/export_output_test.py | 14 ++--- .../saved_model/signature_def_utils_impl.py | 23 ++++++-- 4 files changed, 80 insertions(+), 23 deletions(-) diff --git a/tensorflow/python/estimator/export/export.py b/tensorflow/python/estimator/export/export.py index e2e20f0d71..31e9933c6f 100644 --- a/tensorflow/python/estimator/export/export.py +++ b/tensorflow/python/estimator/export/export.py @@ -33,6 +33,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.util import compat @@ -47,8 +48,8 @@ class ServingInputReceiver(collections.namedtuple( """A return type for a serving_input_receiver_fn. The expected return values are: - features: A dict of string to `Tensor` or `SparseTensor`, specifying the - features to be passed to the model. + features: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or + `SparseTensor`, specifying the features to be passed to the model. receiver_tensors: a `Tensor`, or a dict of string to `Tensor`, specifying input nodes where this receiver expects to be fed by default. Typically, this is a single placeholder expecting serialized `tf.Example` protos. @@ -193,13 +194,14 @@ def build_all_signature_defs(receiver_tensors, raise ValueError('export_outputs must be a dict.') signature_def_map = {} + excluded_signatures = {} for output_key, export_output in export_outputs.items(): signature_name = '{}'.format(output_key or 'None') try: signature = export_output.as_signature_def(receiver_tensors) signature_def_map[signature_name] = signature - except ValueError: - pass + except ValueError as e: + excluded_signatures[signature_name] = str(e) if receiver_tensors_alternatives: for receiver_name, receiver_tensors_alt in ( @@ -213,8 +215,10 @@ def build_all_signature_defs(receiver_tensors, try: signature = export_output.as_signature_def(receiver_tensors_alt) signature_def_map[signature_name] = signature - except ValueError: - pass + except ValueError as e: + excluded_signatures[signature_name] = str(e) + + _log_signature_report(signature_def_map, excluded_signatures) # The above calls to export_output.as_signature_def should return only # valid signatures; if there is a validity problem, they raise ValueError, @@ -224,6 +228,46 @@ def build_all_signature_defs(receiver_tensors, if signature_def_utils.is_valid_signature(v)} +_FRIENDLY_METHOD_NAMES = { + signature_constants.CLASSIFY_METHOD_NAME: 'Classify', + signature_constants.REGRESS_METHOD_NAME: 'Regress', + signature_constants.PREDICT_METHOD_NAME: 'Predict', +} + + +def _log_signature_report(signature_def_map, excluded_signatures): + """Log a report of which signatures were produced.""" + sig_names_by_method_name = collections.defaultdict(list) + + # We'll collect whatever method_names are present, but also we want to make + # sure to output a line for each of the three standard methods even if they + # have no signatures. + for method_name in _FRIENDLY_METHOD_NAMES: + sig_names_by_method_name[method_name] = [] + + for signature_name, sig in signature_def_map.items(): + sig_names_by_method_name[sig.method_name].append(signature_name) + + # TODO(b/67733540): consider printing the full signatures, not just names + for method_name, sig_names in sig_names_by_method_name.items(): + if method_name in _FRIENDLY_METHOD_NAMES: + method_name = _FRIENDLY_METHOD_NAMES[method_name] + logging.info('Signatures INCLUDED in export for {}: {}'.format( + method_name, sig_names if sig_names else 'None')) + + if excluded_signatures: + logging.info('Signatures EXCLUDED from export because they cannot be ' + 'be served via TensorFlow Serving APIs:') + for signature_name, message in excluded_signatures.items(): + logging.info('\'{}\' : {}'.format(signature_name, message)) + + if not signature_def_map: + logging.warn('Export includes no signatures!') + elif (signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY + not in signature_def_map): + logging.warn('Export includes no default signature!') + + # When we create a timestamped directory, there is a small chance that the # directory already exists because another worker is also writing exports. # In this case we just wait one second to get a new timestamp and try again. diff --git a/tensorflow/python/estimator/export/export_output.py b/tensorflow/python/estimator/export/export_output.py index 7c7f92872e..863af6d41d 100644 --- a/tensorflow/python/estimator/export/export_output.py +++ b/tensorflow/python/estimator/export/export_output.py @@ -150,6 +150,9 @@ class RegressionOutput(ExportOutput): return signature_def_utils.regression_signature_def(examples, self.value) +_SINGLE_OUTPUT_DEFAULT_NAME = 'output' + + class PredictOutput(ExportOutput): """Represents the output of a generic prediction head. @@ -162,16 +165,15 @@ class PredictOutput(ExportOutput): """Constructor for PredictOutput. Args: - outputs: A dict of string to `Tensor` representing the predictions. + outputs: A `Tensor` or a dict of string to `Tensor` representing the + predictions. Raises: ValueError: if the outputs is not dict, or any of its keys are not strings, or any of its values are not `Tensor`s. """ if not isinstance(outputs, dict): - raise ValueError( - 'Prediction outputs must be given as a dict of string to Tensor; ' - 'got {}'.format(outputs)) + outputs = {_SINGLE_OUTPUT_DEFAULT_NAME: outputs} for key, value in outputs.items(): if not isinstance(key, six.string_types): raise ValueError( diff --git a/tensorflow/python/estimator/export/export_output_test.py b/tensorflow/python/estimator/export/export_output_test.py index 035a9a143e..7090e53d80 100644 --- a/tensorflow/python/estimator/export/export_output_test.py +++ b/tensorflow/python/estimator/export/export_output_test.py @@ -199,20 +199,18 @@ class ExportOutputTest(test.TestCase): signature_constants.CLASSIFY_METHOD_NAME) self.assertEqual(actual_signature_def, expected_signature_def) - def test_predict_output_constructor(self): - """Tests that no errors are raised when input is expected.""" + def test_predict_outputs_valid(self): + """Tests that no errors are raised when provided outputs are valid.""" outputs = { "output0": constant_op.constant([0]), - u"output1": constant_op.constant([1]), + u"output1": constant_op.constant(["foo"]), } export_output_lib.PredictOutput(outputs) - def test_predict_output_outputs_invalid(self): - with self.assertRaisesRegexp( - ValueError, - "Prediction outputs must be given as a dict of string to Tensor"): - export_output_lib.PredictOutput(constant_op.constant([0])) + # Single Tensor is OK too + export_output_lib.PredictOutput(constant_op.constant([0])) + def test_predict_outputs_invalid(self): with self.assertRaisesRegexp( ValueError, "Prediction output key must be a string"): diff --git a/tensorflow/python/saved_model/signature_def_utils_impl.py b/tensorflow/python/saved_model/signature_def_utils_impl.py index 564befeb0b..240ea61aa5 100644 --- a/tensorflow/python/saved_model/signature_def_utils_impl.py +++ b/tensorflow/python/saved_model/signature_def_utils_impl.py @@ -56,9 +56,13 @@ def build_signature_def(inputs=None, outputs=None, method_name=None): def regression_signature_def(examples, predictions): """Creates regression signature from given examples and predictions. + This function produces signatures intended for use with the TensorFlow Serving + Regress API (tensorflow_serving/apis/prediction_service.proto), and so + constrains the input and output types to those allowed by TensorFlow Serving. + Args: - examples: `Tensor`. - predictions: `Tensor`. + examples: A string `Tensor`, expected to accept serialized tf.Examples. + predictions: A float `Tensor`. Returns: A regression-flavored signature_def. @@ -93,10 +97,15 @@ def regression_signature_def(examples, predictions): def classification_signature_def(examples, classes, scores): """Creates classification signature from given examples and predictions. + This function produces signatures intended for use with the TensorFlow Serving + Classify API (tensorflow_serving/apis/prediction_service.proto), and so + constrains the input and output types to those allowed by TensorFlow Serving. + Args: - examples: `Tensor`. - classes: `Tensor`. - scores: `Tensor`. + examples: A string `Tensor`, expected to accept serialized tf.Examples. + classes: A string `Tensor`. Note that the ClassificationResponse message + requires that class labels are strings, not integers or anything else. + scores: a float `Tensor`. Returns: A classification-flavored signature_def. @@ -140,6 +149,10 @@ def classification_signature_def(examples, classes, scores): def predict_signature_def(inputs, outputs): """Creates prediction signature from given inputs and outputs. + This function produces signatures intended for use with the TensorFlow Serving + Predict API (tensorflow_serving/apis/prediction_service.proto). This API + imposes no constraints on the input and output types. + Args: inputs: dict of string to `Tensor`. outputs: dict of string to `Tensor`. -- GitLab From c2f91136bdfbc0103f01a932566ef46ad4ba9054 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 19 Oct 2017 19:14:44 -0700 Subject: [PATCH 187/573] Fixes build breakage (#13843) --- .../contrib/framework/python/ops/accumulate_n_v2_eager_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py index 8c618838bf..f3453f89fa 100644 --- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py +++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py @@ -79,6 +79,6 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase): if __name__ == "__main__": - eager_context.enable_eager_execution() + ops.enable_eager_execution() test.main() -- GitLab From 932a68370cf3fc076b66f918d1745bce40030f43 Mon Sep 17 00:00:00 2001 From: Mahdi Abavisani Date: Thu, 19 Oct 2017 22:20:44 -0400 Subject: [PATCH 188/573] Update resnet.py (#13828) Test with mnist test set. Previously it was testing on the training set. --- tensorflow/examples/learn/resnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py index 33a09bb6e0..1e0966475b 100755 --- a/tensorflow/examples/learn/resnet.py +++ b/tensorflow/examples/learn/resnet.py @@ -190,8 +190,8 @@ def main(unused_args): # Calculate accuracy. test_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: mnist.train.images}, - y=mnist.train.labels.astype(np.int32), + x={X_FEATURE: mnist.test.images}, + y=mnist.test.labels.astype(np.int32), num_epochs=1, shuffle=False) scores = classifier.evaluate(input_fn=test_input_fn) -- GitLab From bbf1985db1e1b9cddaa04819e29f98f81600f49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jeroen=20B=C3=A9dorf?= Date: Fri, 20 Oct 2017 04:27:14 +0200 Subject: [PATCH 189/573] Fix MPI and Verbs compilation when not using GPUs (#13800) * Fix build errors when building without CUDA support, but with MPI / verbs support * Fix buildifier errors * Moved stream_executor dependency after PR feedback --- tensorflow/core/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 6e434ef49d..013ed2e8fd 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -2179,6 +2179,7 @@ tf_cuda_library( ":lib", ":lib_internal", ":protos_all_cc", + ":stream_executor", "//third_party/eigen3", ] + if_static([":gpu_runtime_impl"]), ) -- GitLab From 58121b8b13597d3285f121f02bd2a512bc76be17 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 20:09:38 -0700 Subject: [PATCH 190/573] Pull out a non-test-only class HloRunnerBase from HloTestBase so that it can be used as a library for running HloModule on given platform. Also add a function to read HloModule from a HloProto file, and a function to make fake input literals for given HloModule. PiperOrigin-RevId: 172835863 --- tensorflow/compiler/xla/service/BUILD | 23 ++ tensorflow/compiler/xla/service/hlo_runner.cc | 199 ++++++++++++++++++ tensorflow/compiler/xla/service/hlo_runner.h | 100 +++++++++ tensorflow/compiler/xla/tests/BUILD | 12 +- .../compiler/xla/tests/hlo_test_base.cc | 114 +--------- tensorflow/compiler/xla/tests/hlo_test_base.h | 26 +-- 6 files changed, 335 insertions(+), 139 deletions(-) create mode 100644 tensorflow/compiler/xla/service/hlo_runner.cc create mode 100644 tensorflow/compiler/xla/service/hlo_runner.h diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 1ef329365e..8f5105aa53 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -2066,6 +2066,29 @@ tf_cc_test( ], ) +cc_library( + name = "hlo_runner", + srcs = ["hlo_runner.cc"], + hdrs = ["hlo_runner.h"], + deps = [ + ":executable", + ":hlo", + ":transfer_manager", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:backend", + "//tensorflow/compiler/xla/service:compiler", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:lib", + "//tensorflow/core:stream_executor_no_cuda", + "//third_party/eigen3", + ], +) + # ----------------------------------------------------------------------------- filegroup( diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc new file mode 100644 index 0000000000..d5d7042a02 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -0,0 +1,199 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_runner.h" + +#include +#include +#include + +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/compiler/xla/layout_util.h" +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/backend.h" +#include "tensorflow/compiler/xla/service/executable.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/transfer_manager.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/common_runtime/eigen_thread_pool.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +namespace se = ::perftools::gputools; + +namespace xla { + +/*static*/ StatusOr> +HloRunner::ReadModuleFromHloProtoFile(const char* filename, + const DebugOptions& debug_options) { + HloProto proto; + TF_RETURN_IF_ERROR(tensorflow::ReadBinaryProto(tensorflow::Env::Default(), + filename, &proto)); + HloModuleConfig config; + config.set_debug_options(debug_options); + TF_ASSIGN_OR_RETURN(auto module, HloModule::CreateFromProto( + proto.hlo_module(), + VersionedComputationHandle(), config)); + return std::move(module); +} + +// Define this in .cc file to avoid having to include eigen or forward declare +// these types in the header. +struct HloRunner::EigenThreadPoolWrapper { + std::unique_ptr pool; + std::unique_ptr device; +}; + +HloRunner::HloRunner() {} + +HloRunner::HloRunner(se::Platform* platform) { + BackendOptions backend_options; + backend_options.set_platform(platform); + backend_ = Backend::CreateBackend(backend_options).ConsumeValueOrDie(); + VLOG(1) << "Created HloRunner for platform: " << platform->Name(); +} + +HloRunner::~HloRunner() { + // Deallocate all the memory allocated during the tests. + for (auto& allocation : allocations_) { + backend().default_stream_executor()->Deallocate(&allocation); + } +} + +StatusOr HloRunner::Execute( + std::unique_ptr module, + tensorflow::gtl::ArraySlice arguments, + Shape* result_shape) { + TF_ASSIGN_OR_RETURN( + std::unique_ptr executable, + backend().compiler()->Compile(std::move(module), + backend().default_stream_executor())); + + se::Stream stream(backend().default_stream_executor()); + stream.Init(); + + ExecutableRunOptions run_options; + run_options.set_stream(&stream); + run_options.set_allocator(backend().memory_allocator()); + run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool()); + run_options.set_intra_op_thread_pool( + backend().eigen_intra_op_thread_pool_device()); + + HloExecutionProfile hlo_execution_profile; + ServiceExecutableRunOptions service_run_options( + run_options, backend().StreamBorrower(), + backend().inter_op_thread_pool()); + TF_ASSIGN_OR_RETURN( + se::DeviceMemoryBase result, + executable->ExecuteOnStream(&service_run_options, arguments, + &hlo_execution_profile)); + TF_RET_CHECK(stream.BlockHostUntilDone()); + + allocations_.push_back(result); + + *result_shape = executable->result_shape(); + + if (ShapeUtil::IsTuple(*result_shape)) { + // We must record element buffers of tuples as well to avoid leaks. + DCHECK(!ShapeUtil::IsNestedTuple(*result_shape)); + TF_ASSIGN_OR_RETURN( + std::vector element_buffers, + backend().transfer_manager()->ShallowCopyTupleFromDevice( + backend().default_stream_executor(), result, *result_shape)); + + // A tuple may contain the same buffer in more than one element. Keep track + // of the buffers already added to avoid duplicates in allocations_. + std::set added_opaques; + for (auto element_buffer : element_buffers) { + if (added_opaques.count(element_buffer.opaque()) == 0) { + CHECK(element_buffer.opaque() != nullptr); + added_opaques.insert(element_buffer.opaque()); + allocations_.push_back(element_buffer); + } + } + } + + return result; +} + +se::DeviceMemoryBase HloRunner::TransferToDevice(const Literal& literal) { + // Allocate memory on the device using the stream executor. + int64 allocation_size = + backend().transfer_manager()->GetByteSizeRequirement(literal.shape()); + se::DeviceMemoryBase allocation = + backend().default_stream_executor()->AllocateArray( + allocation_size); + allocations_.push_back(allocation); + + TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice( + backend().default_stream_executor(), literal, &allocation)); + + return allocation; +} + +std::unique_ptr HloRunner::TransferFromDevice( + const Shape& shape, se::DeviceMemoryBase device_base) { + auto literal = MakeUnique(); + TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice( + backend().default_stream_executor(), device_base, shape, shape, + literal.get())); + return literal; +} + +std::unique_ptr HloRunner::ExecuteAndTransfer( + std::unique_ptr module, + tensorflow::gtl::ArraySlice arguments) { + Shape result_shape; + se::DeviceMemoryBase device_base = + Execute(std::move(module), arguments, &result_shape).ValueOrDie(); + return TransferFromDevice(result_shape, device_base); +} + +template <> +std::unique_ptr HloRunner::Execute( + std::unique_ptr module, + const tensorflow::gtl::ArraySlice>& literals) { + std::vector arguments; + for (const auto& literal : literals) { + arguments.push_back(TransferToDevice(*literal)); + } + return ExecuteAndTransfer(std::move(module), arguments); +} + +template <> +std::unique_ptr HloRunner::Execute( + std::unique_ptr module, + const tensorflow::gtl::ArraySlice& literals) { + std::vector arguments; + for (const auto& literal : literals) { + arguments.push_back(TransferToDevice(*literal)); + } + return ExecuteAndTransfer(std::move(module), arguments); +} + +Backend& HloRunner::backend() { + if (!backend_) { + backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie(); + VLOG(1) << "executing on platform " << backend().platform()->Name(); + } + return *backend_; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h new file mode 100644 index 0000000000..d74a1b59a8 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_runner.h @@ -0,0 +1,100 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ + +#include +#include +#include + +#include "tensorflow/compiler/xla/service/backend.h" +#include "tensorflow/compiler/xla/service/compiler.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/platform/stream_executor_no_cuda.h" + +namespace xla { + +// A base class for running an HloModule. This executes the given HloModule on a +// certain backend directly without using the client interface. HloModule can be +// explicitly built, or loaded from a serialization file (e.g., hlo proto file). +class HloRunner { + public: + HloRunner(); + + HloRunner(::perftools::gputools::Platform* platform); + + ~HloRunner(); + + // Reads the binary proto file in xla.HloProto format, creates and returns the + // HloModule. + static StatusOr> ReadModuleFromHloProtoFile( + const char* filename, const DebugOptions& debug_options); + + // Executes the given module with given literals as input and returns the + // result as a Literal. The LiteralPtr type accepts Literal* or + // std::unique_ptr. + template + std::unique_ptr Execute( + std::unique_ptr module, + const tensorflow::gtl::ArraySlice& literals); + + // Executes the given module and returns a global data handle. + StatusOr Execute( + std::unique_ptr module, + tensorflow::gtl::ArraySlice + arguments, + Shape* result_shape); + + // Transfers the given literal to the device and returns the data handle. + perftools::gputools::DeviceMemoryBase TransferToDevice( + const Literal& literal); + + // Transfers the array referred to by the given handle from the device and + // returns as a Literal. + std::unique_ptr TransferFromDevice( + const Shape& shape, perftools::gputools::DeviceMemoryBase device_base); + + // Executes the given module and return the result as a Literal. + std::unique_ptr ExecuteAndTransfer( + std::unique_ptr module, + tensorflow::gtl::ArraySlice + arguments); + + // If backend is not created in the constructor, creates and returns the + // default backend. If creation fails, crashes the program. + // + // This creates the backend lazily so it's possible to instantiate an + // HloRunner in a program without any backends linked in. + Backend& backend(); + + private: + struct EigenThreadPoolWrapper; + + std::vector allocations_; + + std::unique_ptr thread_pool_wrapper_; + + std::unique_ptr backend_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index b02d906d93..43127925e6 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -102,28 +102,18 @@ cc_library( deps = [ ":literal_test_util", "//tensorflow/compiler/xla:shape_layout", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/legacy_flags:debug_options_flags", - "//tensorflow/compiler/xla/service", "//tensorflow/compiler/xla/service:backend", - "//tensorflow/compiler/xla/service:compiler", "//tensorflow/compiler/xla/service:computation_layout", - "//tensorflow/compiler/xla/service:computation_placer", - "//tensorflow/compiler/xla/service:executable", "//tensorflow/compiler/xla/service:hlo", - "//tensorflow/compiler/xla/service:hlo_execution_profile", - "//tensorflow/compiler/xla/service:hlo_graph_dumper", - "//tensorflow/compiler/xla/service:transfer_manager", - "//tensorflow/core:core_cpu_internal", + "//tensorflow/compiler/xla/service:hlo_runner", "//tensorflow/core:lib", "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/core:test", - "//third_party/eigen3", ], ) diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc index 26513d6ce8..3e244fbfd9 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.cc +++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc @@ -19,24 +19,9 @@ limitations under the License. #include #include -#define EIGEN_USE_THREADS - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h" #include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/compiler/xla/service/backend.h" -#include "tensorflow/compiler/xla/service/computation_layout.h" -#include "tensorflow/compiler/xla/service/executable.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" -#include "tensorflow/compiler/xla/service/hlo_execution_profile.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/transfer_manager.h" -#include "tensorflow/compiler/xla/shape_layout.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" @@ -45,22 +30,6 @@ namespace se = ::perftools::gputools; namespace xla { -// Define this in .cc file to avoid having to include eigen or forward declare -// these types in the header. -struct HloTestBase::EigenThreadPoolWrapper { - std::unique_ptr pool; - std::unique_ptr device; -}; - -HloTestBase::HloTestBase() {} - -HloTestBase::~HloTestBase() { - // Deallocate all the memory allocated during the tests. - for (auto& allocation : allocations_) { - backend().default_stream_executor()->Deallocate(&allocation); - } -} - /* static */ std::unique_ptr HloTestBase::CreateNewModule() { HloModuleConfig config; @@ -80,98 +49,25 @@ StatusOr HloTestBase::Execute( tensorflow::gtl::ArraySlice arguments, Shape* result_shape) { - TF_ASSIGN_OR_RETURN( - std::unique_ptr executable, - backend().compiler()->Compile(std::move(module), - backend().default_stream_executor())); - - se::Stream stream(backend().default_stream_executor()); - stream.Init(); - - ExecutableRunOptions run_options; - run_options.set_stream(&stream); - run_options.set_allocator(backend().memory_allocator()); - run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool()); - run_options.set_intra_op_thread_pool( - backend().eigen_intra_op_thread_pool_device()); - - HloExecutionProfile hlo_execution_profile; - ServiceExecutableRunOptions service_run_options( - run_options, backend().StreamBorrower(), - backend().inter_op_thread_pool()); - TF_ASSIGN_OR_RETURN( - se::DeviceMemoryBase result, - executable->ExecuteOnStream(&service_run_options, arguments, - &hlo_execution_profile)); - TF_RET_CHECK(stream.BlockHostUntilDone()); - - allocations_.push_back(result); - - *result_shape = executable->result_shape(); - - if (ShapeUtil::IsTuple(*result_shape)) { - // We must record element buffers of tuples as well to avoid leaks. - DCHECK(!ShapeUtil::IsNestedTuple(*result_shape)); - TF_ASSIGN_OR_RETURN( - std::vector element_buffers, - backend().transfer_manager()->ShallowCopyTupleFromDevice( - backend().default_stream_executor(), result, *result_shape)); - - // A tuple may contain the same buffer in more than one element. Keep track - // of the buffers already added to avoid duplicates in allocations_. - std::set added_opaques; - for (auto element_buffer : element_buffers) { - if (added_opaques.count(element_buffer.opaque()) == 0) { - CHECK(element_buffer.opaque() != nullptr); - added_opaques.insert(element_buffer.opaque()); - allocations_.push_back(element_buffer); - } - } - } - - return result; + return runner_.Execute(std::move(module), arguments, result_shape); } se::DeviceMemoryBase HloTestBase::TransferToDevice(const Literal& literal) { - // Allocate memory on the device using the stream executor. - int64 allocation_size = - backend().transfer_manager()->GetByteSizeRequirement(literal.shape()); - se::DeviceMemoryBase allocation = - backend().default_stream_executor()->AllocateArray( - allocation_size); - allocations_.push_back(allocation); - - TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice( - backend().default_stream_executor(), literal, &allocation)); - - return allocation; + return runner_.TransferToDevice(literal); } std::unique_ptr HloTestBase::TransferFromDevice( const Shape& shape, se::DeviceMemoryBase device_base) { - auto literal = MakeUnique(); - TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice( - backend().default_stream_executor(), device_base, shape, shape, - literal.get())); - return literal; + return runner_.TransferFromDevice(shape, device_base); } std::unique_ptr HloTestBase::ExecuteAndTransfer( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments) { - Shape result_shape; - se::DeviceMemoryBase device_base = - Execute(std::move(module), arguments, &result_shape).ValueOrDie(); - return TransferFromDevice(result_shape, device_base); + return runner_.ExecuteAndTransfer(std::move(module), arguments); } -Backend& HloTestBase::backend() { - if (!backend_) { - backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie(); - VLOG(1) << "executing on platform " << backend().platform()->Name(); - } - return *backend_; -} +Backend& HloTestBase::backend() { return runner_.backend(); } /* static */ string HloTestBase::TestName() { diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h index 275f1f5c7b..7f068dce36 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.h +++ b/tensorflow/compiler/xla/tests/hlo_test_base.h @@ -21,12 +21,12 @@ limitations under the License. #include #include "tensorflow/compiler/xla/service/backend.h" -#include "tensorflow/compiler/xla/service/compiler.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/computation_layout.h" #include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_runner.h" +#include "tensorflow/compiler/xla/shape_layout.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" -#include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/platform/stream_executor_no_cuda.h" @@ -39,10 +39,9 @@ namespace xla { // building a graph of HLO instructions to run. class HloTestBase : public ::testing::Test { protected: - struct EigenThreadPoolWrapper; - HloTestBase(); + HloTestBase() {} - ~HloTestBase() override; + ~HloTestBase() override {} // Creates a new HLO module for a test. The module created will have // TestName() for its name; it will also automatically populate its debug @@ -102,23 +101,12 @@ class HloTestBase : public ::testing::Test { static string TestName(); - // Creates (if necessary) and returns the default backend. If creation fails, - // crashes the program. - // - // This creates the backend lazily so it's possible to instantiate an - // HloTestBase in a program without any backends linked in. + // Returns the backend owned by the HloRunner. Backend& backend(); - // This vector contains handles of all the device memory allocations performed - // by the test. These are deallocated on destruction of the test object. - std::vector allocations_; + HloRunner runner_; ErrorSpec error_spec_{0.0001}; - - std::unique_ptr thread_pool_wrapper_; - - private: - std::unique_ptr backend_; // Lazily populated. Access via backend(). }; } // namespace xla -- GitLab From aa9ddb2006cba090a53ea978a6ec78bea8245805 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 19 Oct 2017 20:10:03 -0700 Subject: [PATCH 191/573] Add a tool which reads the Hlo module proto and convert it into JSON format. PiperOrigin-RevId: 172835881 --- tensorflow/compiler/xla/tools/BUILD | 12 +++ .../compiler/xla/tools/hlo_proto_to_json.cc | 91 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 tensorflow/compiler/xla/tools/hlo_proto_to_json.cc diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD index 0451537af7..759921dce5 100644 --- a/tensorflow/compiler/xla/tools/BUILD +++ b/tensorflow/compiler/xla/tools/BUILD @@ -210,6 +210,18 @@ tf_cc_binary( ], ) +tf_cc_binary( + name = "hlo_proto_to_json", + srcs = ["hlo_proto_to_json.cc"], + deps = [ + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla/service:hlo_proto", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + ], +) + # ----------------------------------------------------------------------------- filegroup( diff --git a/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc b/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc new file mode 100644 index 0000000000..4e02e17db6 --- /dev/null +++ b/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc @@ -0,0 +1,91 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Usage: +// hlo_proto_to_json --input_file=some_binary_proto +// --output_file=path_to_dump_output +// +// Reads one serilized Hlo module, convert it into JSON format and dump into +// some output directory. some_binaray_proto is obtained by serializing Hlo +// module to disk using --xla_dump_hlo_proto_to debug optoin. + +#include +#include +#include + +#include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/command_line_flags.h" + +using tensorflow::Env; +using xla::string; + +namespace xla { +namespace tools { + +StatusOr ToJson(const tensorflow::protobuf::Message& message) { + string json_output; + tensorflow::protobuf::util::JsonPrintOptions json_options; + json_options.add_whitespace = true; + json_options.always_print_primitive_fields = true; + auto status = tensorflow::protobuf::util::MessageToJsonString( + message, &json_output, json_options); + if (!status.ok()) { + return InternalError("MessageToJsonString failed: %s", + status.error_message().data()); + } + return json_output; +} + +void RealMain(const string& input, const string& output) { + HloProto hlo_proto; + TF_CHECK_OK(tensorflow::ReadBinaryProto(tensorflow::Env::Default(), input, + &hlo_proto)) + << "Can't open, read, or parse input file " << input; + + auto statusor = ToJson(hlo_proto); + QCHECK(statusor.ok()) << "Error converting " << input << " to JSON." + << statusor.status(); + + TF_CHECK_OK(tensorflow::WriteStringToFile(tensorflow::Env::Default(), output, + statusor.ValueOrDie())); +} + +} // namespace tools +} // namespace xla + +int main(int argc, char** argv) { + string input_file, output_file; + const std::vector flag_list = { + tensorflow::Flag("input_file", &input_file, "file to convert."), + tensorflow::Flag("output_file", &output_file, "converted file"), + }; + const string usage = tensorflow::Flags::Usage(argv[0], flag_list); + bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list); + tensorflow::port::InitMain(usage.c_str(), &argc, &argv); + QCHECK(parse_ok && argc == 1) << "\n" << usage; + + QCHECK(!input_file.empty()) << "--input_file is required"; + QCHECK(!output_file.empty()) << "--output_file is required"; + + xla::tools::RealMain(input_file, output_file); + + return 0; +} -- GitLab From 7a1ddf26aed9166af69a560e644abd3f0d4f8ecf Mon Sep 17 00:00:00 2001 From: Sang Han Date: Thu, 19 Oct 2017 20:24:13 -0700 Subject: [PATCH 192/573] Fix casting to size_t for mkl conv filter dims (#13831) --- tensorflow/core/kernels/mkl_conv_ops.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index 57661e8b10..369f632fb4 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -288,8 +288,10 @@ class MklConv2DOp : public OpKernel { mkl_filter_output_mkl_shape.SetMklLayout(mkl_context.prim_fwd, dnnResourceFilter); - size_t filter_sizes[4] = {filter.dim_size(0), filter.dim_size(1), - filter.dim_size(2), filter.dim_size(3)}; + size_t filter_sizes[4] = {static_cast(filter.dim_size(0)), + static_cast(filter.dim_size(1)), + static_cast(filter.dim_size(2)), + static_cast(filter.dim_size(3))}; mkl_filter_output_mkl_shape.SetTfLayout(filter.dims(), filter_sizes, mkl_context.filter_strides); -- GitLab From 513f7df42e4eadfcd241a3be695af6fd426b734e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 19 Oct 2017 20:34:53 -0700 Subject: [PATCH 193/573] Add `int64` out_idx` support for `listdiff`/`list_diff`/`setdiff1d` (#13839) * Add `int64` out_idx` support for `listdiff`/`list_diff`/`setdiff1d` This fix tries to add `int64` `out_idx` support for `listdiff`/`list_diff`/`setdiff1d`. As was specified in docs (`tf.setdiff1d.__doc__`), it is possible to specify `tf.int32` or `tf.int64` for the type of the output idx. However, the `tf.int64` kernel has not been registered. As a consequence, an error will be thrown out if `tf.int64` is used. This fix adds `int64` out_idx` support for `listdiff`/`list_diff`/`setdiff1d` Signed-off-by: Yong Tang * Add template for signature matching of ListDiff kernel. Signed-off-by: Yong Tang * Add test cases for `int64` out_idx support for `tf.listdiff`/`setdiff1d` Signed-off-by: Yong Tang * Add test case for int32 (missed in the last commit) Signed-off-by: Yong Tang --- tensorflow/core/kernels/listdiff_op.cc | 16 ++++++++++----- .../python/kernel_tests/listdiff_op_test.py | 20 ++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tensorflow/core/kernels/listdiff_op.cc b/tensorflow/core/kernels/listdiff_op.cc index d303bdd560..d28a2729d4 100644 --- a/tensorflow/core/kernels/listdiff_op.cc +++ b/tensorflow/core/kernels/listdiff_op.cc @@ -24,12 +24,13 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" namespace tensorflow { -template +template class ListDiffOp : public OpKernel { public: explicit ListDiffOp(OpKernelConstruction* context) : OpKernel(context) { const DataType dt = DataTypeToEnum::v(); - OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, DT_INT32})); + const DataType dtidx = DataTypeToEnum::v(); + OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, dtidx})); } void Compute(OpKernelContext* context) override { @@ -72,9 +73,9 @@ class ListDiffOp : public OpKernel { Tensor* indices = nullptr; OP_REQUIRES_OK(context, context->allocate_output(1, {out_size}, &indices)); - auto Tindices = indices->vec(); + auto Tindices = indices->vec(); - for (int i = 0, p = 0; i < static_cast(x_size); ++i) { + for (Tidx i = 0, p = 0; i < static_cast(x_size); ++i) { if (y_set.count(Tx(i)) == 0) { OP_REQUIRES(context, p < out_size, errors::InvalidArgument( @@ -95,7 +96,12 @@ class ListDiffOp : public OpKernel { .Device(DEVICE_CPU) \ .TypeConstraint("T") \ .TypeConstraint("out_idx"), \ - ListDiffOp) + ListDiffOp) \ + REGISTER_KERNEL_BUILDER(Name("ListDiff") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("out_idx"), \ + ListDiffOp) TF_CALL_REAL_NUMBER_TYPES(REGISTER_LISTDIFF); REGISTER_LISTDIFF(string); diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py index 4f053d2a21..ee86cf0b24 100644 --- a/tensorflow/python/kernel_tests/listdiff_op_test.py +++ b/tensorflow/python/kernel_tests/listdiff_op_test.py @@ -41,15 +41,17 @@ class ListDiffTest(test.TestCase): y = [compat.as_bytes(str(a)) for a in y] out = [compat.as_bytes(str(a)) for a in out] for diff_func in [array_ops.setdiff1d]: - with self.test_session() as sess: - x_tensor = ops.convert_to_tensor(x, dtype=dtype) - y_tensor = ops.convert_to_tensor(y, dtype=dtype) - out_tensor, idx_tensor = diff_func(x_tensor, y_tensor) - tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) - self.assertAllEqual(tf_out, out) - self.assertAllEqual(tf_idx, idx) - self.assertEqual(1, out_tensor.get_shape().ndims) - self.assertEqual(1, idx_tensor.get_shape().ndims) + for index_dtype in [dtypes.int32, dtypes.int64]: + with self.test_session() as sess: + x_tensor = ops.convert_to_tensor(x, dtype=dtype) + y_tensor = ops.convert_to_tensor(y, dtype=dtype) + out_tensor, idx_tensor = diff_func(x_tensor, y_tensor, + index_dtype=index_dtype) + tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) + self.assertAllEqual(tf_out, out) + self.assertAllEqual(tf_idx, idx) + self.assertEqual(1, out_tensor.get_shape().ndims) + self.assertEqual(1, idx_tensor.get_shape().ndims) def testBasic1(self): x = [1, 2, 3, 4] -- GitLab From 492ddb55a9b31a07026b7d82a2f9bcac29f4ee65 Mon Sep 17 00:00:00 2001 From: Suharsh Sivakumar Date: Thu, 19 Oct 2017 21:09:44 -0700 Subject: [PATCH 194/573] Add support for fused batch norm to fake quantize rewriter. PiperOrigin-RevId: 172839124 --- tensorflow/contrib/quantize/BUILD | 33 +- .../quantize/python/copy_graph_test.py | 2 +- .../quantize/python/fold_batch_norms.py | 269 ++++++++++++- .../quantize/python/fold_batch_norms_test.py | 372 ++++++------------ .../contrib/quantize/python/graph_matcher.py | 200 ++++++++++ .../quantize/python/graph_matcher_test.py | 130 ++++++ .../python/quantize_parameterized_test.py | 212 +++++----- 7 files changed, 855 insertions(+), 363 deletions(-) create mode 100644 tensorflow/contrib/quantize/python/graph_matcher.py create mode 100644 tensorflow/contrib/quantize/python/graph_matcher_test.py diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD index 7ff186bc2a..0d6c71965c 100644 --- a/tensorflow/contrib/quantize/BUILD +++ b/tensorflow/contrib/quantize/BUILD @@ -13,6 +13,34 @@ py_library( deps = [], ) +py_library( + name = "graph_matcher", + srcs = [ + "python/graph_matcher.py", + ], + srcs_version = "PY2AND3", + deps = [], +) + +py_test( + name = "graph_matcher_test", + size = "small", + srcs = ["python/graph_matcher_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":graph_matcher", + "//tensorflow/contrib/layers:layers_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:init_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:nn_ops", + "//tensorflow/python:platform_test", + ], +) + py_library( name = "input_to_ops", srcs = ["python/input_to_ops.py"], @@ -43,6 +71,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":common", + ":graph_matcher", ":input_to_ops", "//tensorflow/contrib/graph_editor:graph_editor_py", "//tensorflow/python:array_ops", @@ -58,6 +87,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":fold_batch_norms", + ":graph_matcher", "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", @@ -147,10 +177,11 @@ py_test( py_test( name = "quantize_parameterized_test", - size = "medium", + size = "large", srcs = ["python/quantize_parameterized_test.py"], srcs_version = "PY2AND3", deps = [ + ":fold_batch_norms", ":quantize", "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/contrib/quantize/python/copy_graph_test.py b/tensorflow/contrib/quantize/python/copy_graph_test.py index 0889f12de6..7ff9ad9f84 100644 --- a/tensorflow/contrib/quantize/python/copy_graph_test.py +++ b/tensorflow/contrib/quantize/python/copy_graph_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for tensorflow.quantized.mangle.copy_graph.""" +"""Tests for copy_graph.""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py index c416689510..647d404400 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py @@ -21,7 +21,9 @@ from __future__ import print_function import re from tensorflow.contrib import graph_editor from tensorflow.contrib.quantize.python import common +from tensorflow.contrib.quantize.python import graph_matcher from tensorflow.contrib.quantize.python import input_to_ops +from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn @@ -29,7 +31,7 @@ from tensorflow.python.ops import nn_ops def FoldBatchNorms(graph): - """Finds batch norm layers in the graph, folds them into preceding layers. + """Finds batch norm layers and folds them into preceding layers. Folding only affects the following layers: Conv2D, fully connected, depthwise convolution. @@ -40,10 +42,269 @@ def FoldBatchNorms(graph): Raises: ValueError: When batch norm folding fails. """ - # Fail immediately when the graph contains unsupported fused batch norm ops. - if any(op for op in graph.get_operations() if op.type == 'FusedBatchNorm'): - raise ValueError('Fused batch norm is not supported') + _FoldFusedBatchNorms(graph) + _FoldUnfusedBatchNorms(graph) + +def _FoldFusedBatchNorms(graph): + """Finds fused batch norm layers and folds them into preceding layers. + + Folding only affects the following layers: Conv2D, fully connected, depthwise + convolution. + + Args: + graph: Graph to walk and modify. + + Raises: + ValueError: When batch norm folding fails. + """ + for match in _FindFusedBatchNorms(graph): + scope, sep, _ = match.layer_op.name.rpartition('/') + # Make sure new ops are added to `graph` and put on the same device as + # `bn_op`. The '/' (i.e. `sep`) ensures that we reuse the existing scope + # named `scope`. Otherwise, TF creates a unique scope whose name starts with + # `scope`. + with graph.as_default(), graph.name_scope(scope + sep), ops.device( + match.bn_op.device): + # new weights = old weights * gamma / sqrt(variance + epsilon) + # new biases = -mean * gamma / sqrt(variance + epsilon) + beta + multiplier_tensor = match.gamma_tensor * math_ops.rsqrt( + match.variance_tensor + match.bn_op.get_attr('epsilon')) + bias_tensor = math_ops.subtract( + match.beta_tensor, match.mean_tensor * multiplier_tensor, name='bias') + + # The shape of depthwise weights is different, so we need to reshape the + # multiplier_tensor to ensure that the scaled_weight_tensor has the + # expected shape. + if match.layer_op.type == 'DepthwiseConv2dNative': + new_shape = [ + match.weight_tensor.get_shape().as_list()[2], + match.weight_tensor.get_shape().as_list()[3] + ] + multiplier_tensor = array_ops.reshape( + multiplier_tensor, new_shape, name='scale_reshape') + + # TODO(suharshs): This naming of the following ops needs to carefully + # follow the naming expected by quantize.py. Generalize the quantize code + # to not require these delicate naming conventions. + scaled_weight_tensor = math_ops.multiply( + match.weight_tensor, multiplier_tensor, name='mul_fold') + + new_layer_tensor = _CloneWithNewOperands( + match.layer_op, match.input_tensor, scaled_weight_tensor) + + bias_add_tensor = math_ops.add( + new_layer_tensor, bias_tensor, name='add_fold') + + nodes_modified_count = graph_editor.reroute_ts(bias_add_tensor, + match.output_tensor) + if nodes_modified_count != 1: + raise ValueError( + 'Unexpected inputs to op: %s' % match.output_tensor.name) + + +def _CloneWithNewOperands(layer_op, input_tensor, weight_tensor): + """Clones layer_op with input_tensor and weight_tensor as new inputs.""" + new_layer_name = layer_op.name.split('/')[-1] + '_Fold' + if layer_op.type == 'Conv2D': + return nn_ops.conv2d( + input_tensor, + weight_tensor, + strides=layer_op.get_attr('strides'), + padding=layer_op.get_attr('padding'), + use_cudnn_on_gpu=layer_op.get_attr('use_cudnn_on_gpu'), + data_format=layer_op.get_attr('data_format'), + name=new_layer_name) + elif layer_op.type == 'MatMul': + return math_ops.matmul( + input_tensor, + weight_tensor, + transpose_a=layer_op.get_attr('transpose_a'), + transpose_b=layer_op.get_attr('transpose_b'), + name=new_layer_name) + elif layer_op.type == 'DepthwiseConv2dNative': + return nn.depthwise_conv2d( + input_tensor, + weight_tensor, + strides=layer_op.get_attr('strides'), + padding=layer_op.get_attr('padding'), + name=new_layer_name) + else: + raise ValueError('Cannot handle operation of type: %s' % layer_op.type) + + +def _FindFusedBatchNorms(graph): + """Finds all ops and tensors related to found FusedBatchNorms. + + Args: + graph: Graph to inspect. + + Yields: + _FusedBatchNormMatches. + """ + input_pattern = graph_matcher.OpTypePattern('*') + weight_pattern = graph_matcher.OpTypePattern('*') + gamma_pattern = graph_matcher.OpTypePattern('*') + beta_pattern = graph_matcher.OpTypePattern('*') + mean_pattern = graph_matcher.OpTypePattern('*') + variance_pattern = graph_matcher.OpTypePattern('*') + + conv_pattern = graph_matcher.OpTypePattern( + 'Conv2D|DepthwiseConv2dNative', inputs=[input_pattern, weight_pattern]) + # MatMul has a Reshape between it and FusedBatchNorm. + matmul_pattern = graph_matcher.OpTypePattern( + 'MatMul', inputs=[input_pattern, weight_pattern]) + matmul_reshape_pattern = graph_matcher.OpTypePattern( + 'Reshape', inputs=[matmul_pattern, + graph_matcher.OpTypePattern('*')]) + + conv_batch_norm_pattern = graph_matcher.OpTypePattern( + 'FusedBatchNorm', + inputs=[ + conv_pattern, gamma_pattern, beta_pattern, mean_pattern, + variance_pattern + ]) + matmul_batch_norm_pattern = graph_matcher.OpTypePattern( + 'FusedBatchNorm', + inputs=[ + matmul_reshape_pattern, gamma_pattern, beta_pattern, mean_pattern, + variance_pattern + ]) + matmul_bn_output_reshape_pattern = graph_matcher.OpTypePattern( + 'Reshape', + inputs=[matmul_batch_norm_pattern, + graph_matcher.OpTypePattern('*')]) + + conv_matcher = graph_matcher.GraphMatcher(conv_batch_norm_pattern) + matmul_matcher = graph_matcher.GraphMatcher(matmul_bn_output_reshape_pattern) + + def _GetCommonTensors(match_result): + """Gets tensors needed for FusedBatchNormMatch from match_result.""" + input_tensor = match_result.get_tensor(input_pattern) + weight_tensor = match_result.get_tensor(weight_pattern) + gamma_tensor = match_result.get_tensor(gamma_pattern) + beta_tensor = match_result.get_tensor(beta_pattern) + # FusedBatchNorm in training is different from that in inference. It takes + # empty 'mean' and empty 'variance', and produces the mean and the variance + # of the batch. Therefore, when is_training is true, mean_tensor and + # variance_tensor point to 1st and 2nd (0-based) output of bn_op, + # respectively; when is_training is false, they point to bn_op's inputs. + is_training = bn_op.get_attr('is_training') + if is_training: + mean_tensor = bn_op.outputs[1] + variance_tensor = bn_op.outputs[2] + else: + mean_tensor = match_result.get_tensor(mean_pattern) + variance_tensor = match_result.get_tensor(variance_pattern) + return (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor) + + for match_result in conv_matcher.match_graph(graph): + layer_op = match_result.get_op(conv_pattern) + bn_op = match_result.get_op(conv_batch_norm_pattern) + # In the case of convolution the output_tensor is the output of bn_op. + output_tensor = bn_op.outputs[0] + + (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor) = _GetCommonTensors(match_result) + yield _FusedBatchNormMatch( + layer_op=layer_op, + bn_op=bn_op, + output_tensor=output_tensor, + input_tensor=input_tensor, + weight_tensor=weight_tensor, + gamma_tensor=gamma_tensor, + beta_tensor=beta_tensor, + mean_tensor=mean_tensor, + variance_tensor=variance_tensor) + + for match_result in matmul_matcher.match_graph(graph): + layer_op = match_result.get_op(matmul_pattern) + bn_op = match_result.get_op(matmul_batch_norm_pattern) + # In the MatMul case, the output of batch norm is reshaped back into a + # 2D tensor, so the output_tensor is the output of the Reshape op. + output_reshape_op = match_result.get_op(matmul_bn_output_reshape_pattern) + output_tensor = output_reshape_op.outputs[0] + + (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor) = _GetCommonTensors(match_result) + yield _FusedBatchNormMatch( + layer_op=layer_op, + bn_op=bn_op, + output_tensor=output_tensor, + input_tensor=input_tensor, + weight_tensor=weight_tensor, + gamma_tensor=gamma_tensor, + beta_tensor=beta_tensor, + mean_tensor=mean_tensor, + variance_tensor=variance_tensor) + + +class _FusedBatchNormMatch(object): + """Contains all information related to a found FusedBatchNorm.""" + + def __init__(self, layer_op, bn_op, output_tensor, input_tensor, + weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor): + self._layer_op = layer_op + self._bn_op = bn_op + self._output_tensor = output_tensor + self._input_tensor = input_tensor + self._weight_tensor = weight_tensor + self._gamma_tensor = gamma_tensor + self._beta_tensor = beta_tensor + self._mean_tensor = mean_tensor + self._variance_tensor = variance_tensor + + @property + def layer_op(self): + return self._layer_op + + @property + def bn_op(self): + return self._bn_op + + @property + def output_tensor(self): + return self._output_tensor + + @property + def input_tensor(self): + return self._input_tensor + + @property + def weight_tensor(self): + return self._weight_tensor + + @property + def gamma_tensor(self): + return self._gamma_tensor + + @property + def beta_tensor(self): + return self._beta_tensor + + @property + def mean_tensor(self): + return self._mean_tensor + + @property + def variance_tensor(self): + return self._variance_tensor + + +def _FoldUnfusedBatchNorms(graph): + """Finds unfused batch norm layers and folds them into preceding layers. + + Folding only affects the following layers: Conv2D, fully connected, depthwise + convolution. + + Args: + graph: Graph to walk and modify. + + Raises: + ValueError: When batch norm folding fails. + """ input_to_ops_map = input_to_ops.InputToOps(graph) for bn in common.BatchNormGroups(graph): diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py index ddedb0a2c0..5a66b38b15 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import copy from tensorflow.contrib.layers.python.layers import layers from tensorflow.contrib.quantize.python import fold_batch_norms from tensorflow.python.framework import dtypes @@ -35,57 +34,32 @@ conv2d = layers.conv2d fully_connected = layers.fully_connected separable_conv2d = layers.separable_conv2d -_DEFAULT_BATCH_NORM_PARAMS = { - 'center': True, - 'scale': True, - 'decay': 1.0 - 0.003, - 'fused': False, -} - # TODO(suharshs): Use parameterized test once OSS TF supports it. class FoldBatchNormsTest(test_util.TensorFlowTestCase): def _RunTestOverParameters(self, test_fn): parameters_list = [ - # (relu, relu_op_name, with_bypass) - (nn_ops.relu6, 'Relu6', False), - (nn_ops.relu, 'Relu', False), - (nn_ops.relu6, 'Relu6', True), - (nn_ops.relu, 'Relu', True), + # (relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm) + (nn_ops.relu6, 'Relu6', False, False, False), + (nn_ops.relu, 'Relu', False, False, False), + (nn_ops.relu6, 'Relu6', True, False, False), + (nn_ops.relu, 'Relu', True, False, False), + (nn_ops.relu6, 'Relu6', False, True, False), + (nn_ops.relu, 'Relu', False, True, False), + (nn_ops.relu6, 'Relu6', True, True, False), + (nn_ops.relu, 'Relu', True, True, False), + # Fused batch norm always has scaling enabled. + (nn_ops.relu6, 'Relu6', False, True, True), + (nn_ops.relu, 'Relu', False, True, True), + (nn_ops.relu6, 'Relu6', True, True, True), + (nn_ops.relu, 'Relu', True, True, True), ] - for parameters in parameters_list: - test_fn(parameters[0], parameters[1], parameters[2]) - - def testFailsWithFusedBatchNorm(self): - self._RunTestOverParameters(self._TestFailsWithFusedBatchNorm) + for params in parameters_list: + test_fn(params[0], params[1], params[2], params[3], params[4]) - def _TestFailsWithFusedBatchNorm(self, relu, relu_op_name, with_bypass): - """Tests that batch norm fails when fused batch norm ops are present.""" - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - out_depth = 3 if with_bypass else 32 - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - batch_norm_params = _DEFAULT_BATCH_NORM_PARAMS.copy() - batch_norm_params['fused'] = True - scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=batch_norm_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - with self.assertRaises(ValueError): - fold_batch_norms.FoldBatchNorms(g) - - def _TestFoldConv2d(self, relu, relu_op_name, with_bypass): + def _TestFoldConv2d(self, relu, relu_op_name, with_bypass, has_scaling, + fused_batch_norm): """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. Args: @@ -93,6 +67,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -102,12 +78,17 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): stride = 1 if with_bypass else 2 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) + node = conv2d( + inputs, + out_depth, [5, 5], + stride=stride, + padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), + scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') relu(node, name='test/' + relu_op_name) @@ -116,9 +97,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/mul']) + self._AssertInputOpsAre(folded_mul, [ + scope + '/weights/read', + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) + ]) self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') @@ -129,16 +111,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/convolution_Fold', - scope + '/BatchNorm/batchnorm/sub']) + self._AssertInputOpsAre(folded_add, [ + scope + '/convolution_Fold', + self._BathNormBiasName(scope, fused_batch_norm) + ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) def testFoldConv2d(self): self._RunTestOverParameters(self._TestFoldConv2d) - def _TestFoldConv2dUnknownShape(self, relu, relu_op_name, with_bypass): + def _TestFoldConv2dUnknownShape(self, relu, relu_op_name, with_bypass, + has_scaling, fused_batch_norm): """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. Tests that folding works even with an input shape where some dimensions are @@ -149,6 +133,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -165,7 +151,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): weights_initializer=self._WeightInit(0.09), activation_fn=activation_fn, normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') @@ -176,7 +163,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') self._AssertInputOpsAre(folded_mul, [ - scope + '/weights/read', scope + '/BatchNorm/batchnorm/mul' + scope + '/weights/read', + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) ]) self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) @@ -188,7 +176,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') self._AssertInputOpsAre(folded_add, [ - scope + '/convolution_Fold', scope + '/BatchNorm/batchnorm/sub' + scope + '/convolution_Fold', + self._BathNormBiasName(scope, fused_batch_norm) ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) @@ -196,62 +185,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): def testFoldConv2dUnknownShape(self): self._RunTestOverParameters(self._TestFoldConv2dUnknownShape) - def _TestFoldConv2dWithoutScale(self, relu, relu_op_name, with_bypass): - """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - out_depth = 3 if with_bypass else 32 - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS) - bn_params['scale'] = False - scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms(g) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/Rsqrt']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) - - folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') - self.assertEqual(folded_conv.type, 'Conv2D') - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/convolution_Fold', - scope + '/BatchNorm/batchnorm/sub']) - output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - - def testFoldConv2dWithoutScale(self): - self._RunTestOverParameters(self._TestFoldConv2dWithoutScale) - - def _TestFoldFullyConnectedLayer(self, relu, relu_op_name, with_bypass): + def _TestFoldFullyConnectedLayer(self, relu, relu_op_name, with_bypass, + has_scaling, fused_batch_norm): """Tests folding cases: inputs -> FC with batch norm -> Relu*. Args: @@ -259,6 +194,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -267,12 +204,15 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): out_depth = 256 if with_bypass else 128 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' - node = fully_connected(inputs, out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) + node = fully_connected( + inputs, + out_depth, + weights_initializer=self._WeightInit(0.03), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), + scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') relu(node, name='test/' + relu_op_name) @@ -281,9 +221,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/mul']) + self._AssertInputOpsAre(folded_mul, [ + scope + '/weights/read', + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) + ]) self._AssertOutputGoesToOps(folded_mul, g, [scope + '/MatMul_Fold']) folded_conv = g.get_operation_by_name(scope + '/MatMul_Fold') @@ -294,71 +235,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/MatMul_Fold', - scope + '/BatchNorm/batchnorm/sub']) + self._AssertInputOpsAre(folded_add, [ + scope + '/MatMul_Fold', + self._BathNormBiasName(scope, fused_batch_norm) + ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) def testFoldFullyConnectedLayer(self): self._RunTestOverParameters(self._TestFoldFullyConnectedLayer) - def _TestFoldFullyConnectedLayerWithoutScale(self, relu, relu_op_name, - with_bypass): - """Tests folding cases: inputs -> FC with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - """ - g = ops.Graph() - with g.as_default(): - batch_size, depth = 5, 256 - inputs = array_ops.zeros((batch_size, depth)) - out_depth = 256 if with_bypass else 128 - activation_fn = None if with_bypass else relu - bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS) - bn_params['scale'] = False - scope = 'test/test2' if with_bypass else 'test' - node = fully_connected(inputs, out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms(g) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/Rsqrt']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/MatMul_Fold']) - - folded_conv = g.get_operation_by_name(scope + '/MatMul_Fold') - self.assertEqual(folded_conv.type, 'MatMul') - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/MatMul_Fold', - scope + '/BatchNorm/batchnorm/sub']) - output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - - def testFoldFullyConnectedLayerWithoutScale(self): - self._RunTestOverParameters(self._TestFoldFullyConnectedLayerWithoutScale) - - def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass): + def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass, + has_scaling, fused_batch_norm): """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*. Args: @@ -366,6 +254,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -374,13 +264,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): stride = 1 if with_bypass else 2 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' - node = separable_conv2d(inputs, None, [5, 5], stride=stride, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) + node = separable_conv2d( + inputs, + None, [5, 5], + stride=stride, + depth_multiplier=1.0, + padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), + scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') relu(node, name='test/' + relu_op_name) @@ -396,9 +291,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): scale_reshape = g.get_operation_by_name(scope + '/scale_reshape') self.assertEqual(scale_reshape.type, 'Reshape') - self._AssertInputOpsAre(scale_reshape, - [scope + '/BatchNorm/batchnorm/mul', - scope + '/scale_reshape/shape']) + self._AssertInputOpsAre(scale_reshape, [ + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm), + scope + '/scale_reshape/shape' + ]) self._AssertOutputGoesToOps(scale_reshape, g, [scope + '/mul_fold']) folded_conv = g.get_operation_by_name(scope + '/depthwise_Fold') @@ -409,77 +305,35 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/depthwise_Fold', - scope + '/BatchNorm/batchnorm/sub']) + self._AssertInputOpsAre(folded_add, [ + scope + '/depthwise_Fold', + self._BathNormBiasName(scope, fused_batch_norm) + ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) def testFoldDepthwiseConv2d(self): self._RunTestOverParameters(self._TestFoldDepthwiseConv2d) - def _TestFoldDepthwiseConv2dWithoutScale(self, relu, relu_op_name, - with_bypass): - """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS) - bn_params['scale'] = False - scope = 'test/test2' if with_bypass else 'test' - node = separable_conv2d(inputs, None, [5, 5], stride=stride, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms(g) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/depthwise_weights/read', - scope + '/scale_reshape']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/depthwise_Fold']) - - scale_reshape = g.get_operation_by_name(scope + '/scale_reshape') - self.assertEqual(scale_reshape.type, 'Reshape') - self._AssertInputOpsAre(scale_reshape, - [scope + '/BatchNorm/batchnorm/Rsqrt', - scope + '/scale_reshape/shape']) - self._AssertOutputGoesToOps(scale_reshape, g, [scope + '/mul_fold']) - - folded_conv = g.get_operation_by_name(scope + '/depthwise_Fold') - self.assertEqual(folded_conv.type, 'DepthwiseConv2dNative') - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/depthwise_Fold', - scope + '/BatchNorm/batchnorm/sub']) - output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - - def testFoldDepthwiseConv2dWithoutScale(self): - self._RunTestOverParameters(self._TestFoldDepthwiseConv2dWithoutScale) + def _BatchNormParams(self, scale=True, fused=False): + return { + 'center': True, + 'scale': scale, + 'decay': 1.0 - 0.003, + 'fused': fused + } + + def _BatchNormMultiplierName(self, scope, has_scaling, fused): + if has_scaling: + if fused: + return scope + '/mul' + return scope + '/BatchNorm/batchnorm/mul' + return scope + '/BatchNorm/batchnorm/Rsqrt' + + def _BathNormBiasName(self, scope, fused): + if fused: + return scope + '/bias' + return scope + '/BatchNorm/batchnorm/sub' def _WeightInit(self, stddev): """Returns a truncated normal variable initializer. diff --git a/tensorflow/contrib/quantize/python/graph_matcher.py b/tensorflow/contrib/quantize/python/graph_matcher.py new file mode 100644 index 0000000000..e3581cc559 --- /dev/null +++ b/tensorflow/contrib/quantize/python/graph_matcher.py @@ -0,0 +1,200 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities that match patterns in a tf.Graph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class OpTypePattern(object): + """A tree pattern that matches TF expressions with certain op types.""" + + def __init__(self, op_type, name=None, inputs=None): + """Initializes an OpTypePattern. + + Args: + op_type: string that specifies the allowed types of the root. It can be + (1) an op type, e.g. 'Conv2D', + (2) '*', i.e. wildcard, or + (3) multiple op types separated by '|', e.g., 'Relu|Relu6'. + We could use regex strings, which might be worthwhile when we have many + similar TF op types. + name: Optional string. The name of the pattern that can be looked up in + MatchResult. + inputs: Optional list of `OpTypePattern`s or strings that specify the + patterns for the inputs of a matching op. If None, this pattern accepts + any inputs of a matching op. + """ + self._op_type = op_type + self._name = name + if inputs is None: + inputs = [] + self._inputs = [ + input_pattern if isinstance(input_pattern, OpTypePattern) else + OpTypePattern(input_pattern) for input_pattern in inputs + ] + + @property + def op_type(self): + return self._op_type + + @property + def inputs(self): + return self._inputs + + @property + def name(self): + return self._name + + +class MatchResult(object): + r"""Encapsulates the result of a match done by GraphMatcher. + + MatchResult contains a map from OpTypePattern to the matching op and tensor. + When the matching op has multiple output tensors, the matching tensor is the + output tensor used by the matching op of the parent pattern. E.g., when we + match graph + + - + + / \y0 y1/ \ + x split z + | + y (nodes are ops; edges are going up) + + against add_pattern defined as + + y1_pattern = OpTypePattern('*') + z_pattern = OpTypePattern('*') + add_pattern = OpTypePattern('+', inputs=[y1_pattern, z_pattern]) + + the matching op of `y1_pattern` is `split`, and the matching tensor of + `y1_pattern` + is `y1` not `y0`. + """ + + def __init__(self): + self._pattern_to_op_tensor = {} + self._name_to_pattern = {} + + def add(self, pattern, op, tensor): + self._pattern_to_op_tensor[pattern] = op, tensor + if pattern.name is not None: + if pattern.name in self._name_to_pattern: + raise ValueError( + 'Name %s is already bound to another pattern' % pattern.name) + self._name_to_pattern[pattern.name] = pattern + + def _to_pattern(self, pattern_or_name): + if isinstance(pattern_or_name, OpTypePattern): + return pattern_or_name + + if isinstance(pattern_or_name, str): + return self._name_to_pattern[pattern_or_name] + + raise ValueError('pattern_or_name has type %s. Expect OpTypePattern or str.' + % type(pattern_or_name)) + + def get_op(self, pattern_or_name): + return self._pattern_to_op_tensor[self._to_pattern(pattern_or_name)][0] + + def get_tensor(self, pattern_or_name): + return self._pattern_to_op_tensor[self._to_pattern(pattern_or_name)][1] + + +class GraphMatcher(object): + """Checks if a particular subgraph matches a given pattern.""" + + def __init__(self, pattern): + """Initializes a GraphMatcher. + + Args: + pattern: The `OpTypePattern` against which `GraphMatcher` matches + subgraphs. + """ + self._pattern = pattern + + def _match_pattern(self, pattern, op, tensor): + """Returns whether an TF expression rooted at `op` matches `pattern`. + + If there is a match, adds to `self._match_result` the matching op and tensor + with key `pattern`. + + Args: + pattern: An `OpTypePattern`. + op: A `tf.Operation` to match against the pattern. + tensor: the output `tf.Tensor` of `op` that is used by the matching op of + `pattern`'s parent. Can be None if `pattern` is already the root of the + pattern tree. + + Returns: + True if an TF expression rooted at `op` matches `pattern`. + """ + if pattern.op_type != '*': + if op.type not in pattern.op_type.split('|'): + return False + + self._match_result.add(pattern, op, tensor) + + if not pattern.inputs: + # If pattern.inputs is empty, skips the rest and accepts all the inputs. + return True + + return len(op.inputs) == len(pattern.inputs) and all([ + self._match_pattern(input_pattern, input_tensor.op, input_tensor) + for input_tensor, input_pattern in zip(op.inputs, pattern.inputs) + ]) + + def match_op(self, op): + """Matches `op` against `self._pattern`. + + Args: + op: `tf.Operation` to match against the pattern. + + Returns: + Returns a `MatchResult` if `op` matches the pattern; otherwise, returns + None. + """ + self._match_result = MatchResult() + if not self._match_pattern(self._pattern, op, tensor=None): + return None + return self._match_result + + def match_ops(self, ops): + """Matches each operation in `ops` against `self._pattern`. + + Args: + ops: collection of `tf.Operation` to match against the pattern. + + Yields: + `MatchResult` for each `tf.Operation` that matches the pattern. + """ + for op in ops: + match_result = self.match_op(op) + if match_result: + yield match_result + + def match_graph(self, graph): + """Matches each operation in `graph` against `self._pattern`. + + Args: + graph: `tf.Graph` containing operations to match. + + Yields: + `MatchResult` for each `tf.Operation` in `graph` that matches the pattern. + """ + # Python 3.3.2+ implements `yield from`, but for now: + for match_result in self.match_ops(graph.get_operations()): + yield match_result diff --git a/tensorflow/contrib/quantize/python/graph_matcher_test.py b/tensorflow/contrib/quantize/python/graph_matcher_test.py new file mode 100644 index 0000000000..e1572865e4 --- /dev/null +++ b/tensorflow/contrib/quantize/python/graph_matcher_test.py @@ -0,0 +1,130 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for graph_matcher.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.framework.python import ops as contrib_ops +from tensorflow.contrib.layers.python.layers import initializers +from tensorflow.contrib.layers.python.layers import layers +from tensorflow.contrib.quantize.python import graph_matcher +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import googletest + + +class GraphMatcherTest(test_util.TensorFlowTestCase): + + def test_conv_layer(self): + g = ops.Graph() + with g.as_default(): + inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3]) + + with contrib_ops.arg_scope( + [layers.batch_norm], fused=True, is_training=True, trainable=True): + return layers.convolution( + inputs, + num_outputs=16, + kernel_size=3, + stride=1, + padding='VALID', + activation_fn=nn_ops.relu, + normalizer_fn=layers.batch_norm, + normalizer_params={}, + weights_initializer=initializers.xavier_initializer(), + weights_regularizer=None, + biases_initializer=init_ops.zeros_initializer(), + biases_regularizer=None, + reuse=None, + trainable=True, + scope=None) + + inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs') + relu_pattern = graph_matcher.OpTypePattern( + 'Relu', + name='relu', + inputs=[ + graph_matcher.OpTypePattern( + 'FusedBatchNorm', + inputs=[ + graph_matcher.OpTypePattern( + 'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', '*', + '*' + ]) + ]) + matcher = graph_matcher.GraphMatcher(relu_pattern) + match_results = list(matcher.match_graph(g)) + self.assertEqual(1, len(match_results)) + match_result = match_results[0] + self.assertEqual(match_result.get_tensor(inputs_pattern), inputs) + self.assertEqual(match_result.get_tensor('inputs'), inputs) + + def test_multiple_outputs(self): + # - + + # / \y0 y1/ \ + # x split z + # | + # y (nodes are ops; edges are going up) + g = ops.Graph() + with g.as_default(): + x = array_ops.placeholder(dtypes.float32, shape=[1], name='x') + y = array_ops.placeholder(dtypes.float32, shape=[2], name='y') + y0, y1 = array_ops.split(y, num_or_size_splits=2, axis=0) + z = array_ops.placeholder(dtypes.float32, shape=[1], name='z') + math_ops.add(x, y0) + math_ops.subtract(y1, z) + + y1_pattern = graph_matcher.OpTypePattern('*') + minus_pattern = graph_matcher.OpTypePattern('Sub', inputs=[y1_pattern, '*']) + matcher = graph_matcher.GraphMatcher(minus_pattern) + + match_results = list(matcher.match_graph(g)) + self.assertEqual(1, len(match_results)) + match_result = match_results[0] + + self.assertEqual(y0.op, y1.op) + self.assertEqual(match_result.get_op(y1_pattern), y1.op) + self.assertEqual(match_result.get_tensor(y1_pattern), y1) + + def test_oneof_pattern(self): + # - + + # / \ / \ + # x y z + g = ops.Graph() + with g.as_default(): + x = array_ops.placeholder(dtypes.float32, shape=[], name='x') + y = array_ops.placeholder(dtypes.float32, shape=[], name='y') + z = array_ops.placeholder(dtypes.float32, shape=[], name='z') + plus = x + y + minus = y - z + + add_or_sub_pattern = graph_matcher.OpTypePattern( + 'Add|Sub', inputs=['*', '*']) + matcher = graph_matcher.GraphMatcher(add_or_sub_pattern) + self.assertEqual([ + match_result.get_op(add_or_sub_pattern) + for match_result in matcher.match_graph(g) + ], [plus.op, minus.op]) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py index b5a32a7266..31fcd66dfb 100644 --- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py +++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.layers.python.layers import layers +from tensorflow.contrib.quantize.python import fold_batch_norms from tensorflow.contrib.quantize.python import quantize from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -35,18 +36,11 @@ conv2d = layers.conv2d fully_connected = layers.fully_connected separable_conv2d = layers.separable_conv2d -_DEFAULT_BATCH_NORM_PARAMS = { - 'center': True, - 'scale': True, - 'decay': 1.0 - 0.003, - 'fused': False, -} - -# TODO(suharshs): Use parameterized test once OSS TF supports it. class QuantizeTest(test_util.TensorFlowTestCase): - def _RunTestOverParameters(self, test_fn): + def _RunWithoutBatchNormTestOverParameters(self, test_fn): + # TODO(suharshs): Use parameterized test once OSS TF supports it. parameters_list = [ # (activation, activation_op_name, with_bypass, delay) (nn_ops.relu6, 'Relu6', False, None), @@ -60,10 +54,10 @@ class QuantizeTest(test_util.TensorFlowTestCase): (array_ops.identity, 'Identity', True, None), (nn_ops.relu6, 'Relu6', True, 5000), (nn_ops.relu, 'Relu', True, 5000), - (array_ops.identity, 'Identity', True, 5000) + (array_ops.identity, 'Identity', True, 5000), ] - for parameters in parameters_list: - test_fn(parameters[0], parameters[1], parameters[2], parameters[3]) + for params in parameters_list: + test_fn(params[0], params[1], params[2], params[3]) def _TestQuantize_Conv2dWithoutBatchNorm(self, activation, activation_op_name, with_bypass, delay): @@ -137,7 +131,8 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def testQuantize_Conv2dWithoutBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_Conv2dWithoutBatchNorm) + self._RunWithoutBatchNormTestOverParameters( + self._TestQuantize_Conv2dWithoutBatchNorm) def _TestQuantize_FCWithoutBatchNorm(self, activation, activation_op_name, with_bypass, delay): @@ -210,7 +205,8 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def testQuantize_FCWithoutBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_FCWithoutBatchNorm) + self._RunWithoutBatchNormTestOverParameters( + self._TestQuantize_FCWithoutBatchNorm) def _TestQuantize_DepthwiseConv2dWithoutBatchNorm( self, activation, activation_op_name, with_bypass, delay): @@ -284,11 +280,43 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def testQuantize_DepthwiseConv2dWithoutBatchNorm(self): - self._RunTestOverParameters( + self._RunWithoutBatchNormTestOverParameters( self._TestQuantize_DepthwiseConv2dWithoutBatchNorm) + def _RunBatchNormTestOverParameters(self, test_fn): + # TODO(suharshs): Use parameterized test once OSS TF supports it. + parameters_list = [ + # (activation, activation_op_name, with_bypass, delay, fused_batch_norm) + (nn_ops.relu6, 'Relu6', False, None, False), + (nn_ops.relu, 'Relu', False, None, False), + (array_ops.identity, 'Identity', False, None, False), + (nn_ops.relu6, 'Relu6', False, 5000, False), + (nn_ops.relu, 'Relu', False, 5000, False), + (array_ops.identity, 'Identity', False, 5000, False), + (nn_ops.relu6, 'Relu6', True, None, False), + (nn_ops.relu, 'Relu', True, None, False), + (array_ops.identity, 'Identity', True, None, False), + (nn_ops.relu6, 'Relu6', True, 5000, False), + (nn_ops.relu, 'Relu', True, 5000, False), + (array_ops.identity, 'Identity', True, 5000, False), + (nn_ops.relu6, 'Relu6', False, None, True), + (nn_ops.relu, 'Relu', False, None, True), + (array_ops.identity, 'Identity', False, None, True), + (nn_ops.relu6, 'Relu6', False, 5000, True), + (nn_ops.relu, 'Relu', False, 5000, True), + (array_ops.identity, 'Identity', False, 5000, True), + (nn_ops.relu6, 'Relu6', True, None, True), + (nn_ops.relu, 'Relu', True, None, True), + (array_ops.identity, 'Identity', True, None, True), + (nn_ops.relu6, 'Relu6', True, 5000, True), + (nn_ops.relu, 'Relu', True, 5000, True), + (array_ops.identity, 'Identity', True, 5000, True) + ] + for params in parameters_list: + test_fn(params[0], params[1], params[2], params[3], params[4]) + def _TestQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay): + with_bypass, delay, fused_batch_norm): """Tests quantization: inputs -> Conv2d with batch norm -> Activation. Args: @@ -298,25 +326,29 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. """ self._testQuantize_Conv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=True) self._testQuantize_Conv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=False) def testQuantize_Conv2dWithBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_Conv2dWithBatchNorm) + self._RunBatchNormTestOverParameters(self._TestQuantize_Conv2dWithBatchNorm) def _testQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay, use_ema): + with_bypass, delay, fused_batch_norm, + use_ema): """Tests quantization: inputs -> Conv2d with batch norm -> Activation. Args: @@ -326,6 +358,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. use_ema: Bool, when true uses EMA quantization for BN folded weights. """ graph = ops.Graph() @@ -337,39 +370,29 @@ class QuantizeTest(test_util.TensorFlowTestCase): stride = 1 if with_bypass else 2 out_depth = 3 if with_bypass else 32 scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) - # Manually fold the batch norm. - weights = graph.get_operation_by_name(scope + '/weights/read').outputs[0] - bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul') - .outputs[0]) - mul_fold = math_ops.multiply(weights, bn_mult, name=scope + '/mul_fold') - stride = [stride, stride] - conv_fold = nn_ops.convolution( - input=inputs, - filter=mul_fold, + node = conv2d( + inputs, + out_depth, [5, 5], + stride=stride, padding='SAME', - strides=stride, - data_format='NHWC', - name=scope + '/convolution_Fold') - bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub') - .outputs[0]) - add_fold = math_ops.add(conv_fold, bn_bias, name=scope + '/add_fold') + weights_initializer=self._WeightInit(0.09), + activation_fn=None, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams(fused_batch_norm), + scope=scope) + # Manually add a bypass (optionaly) and an activation. if with_bypass: - node = math_ops.add(inputs, add_fold, name='test/Add') - else: - node = add_fold + node = math_ops.add(inputs, node, name='test/Add') + node = activation(node, name='test/' + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') + fold_batch_norms.FoldBatchNorms(graph) + quantize.Quantize( graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema) @@ -413,7 +436,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def _TestQuantize_FCWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay): + with_bypass, delay, fused_batch_norm): """Tests quantization: inputs -> FC with batch norm -> Activation. Args: @@ -423,25 +446,29 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. """ self._testQuantize_FCWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=True) self._testQuantize_FCWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=False) def testQuantize_FCWithBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_FCWithBatchNorm) + self._RunBatchNormTestOverParameters(self._TestQuantize_FCWithBatchNorm) def _testQuantize_FCWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay, use_ema): + with_bypass, delay, fused_batch_norm, + use_ema): """Tests quantization: inputs -> FC with batch norm -> Activation. Args: @@ -451,6 +478,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. use_ema: Bool, when true uses EMA quantization for BN folded weights. """ graph = ops.Graph() @@ -461,32 +489,27 @@ class QuantizeTest(test_util.TensorFlowTestCase): inputs = array_ops.zeros((batch_size, depth)) out_depth = 256 if with_bypass else 128 scope = 'test/test2' if with_bypass else 'test' - node = fully_connected(inputs, out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) - # Manually fold the batch norm. - weights = graph.get_operation_by_name(scope + '/weights/read').outputs[0] - bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul') - .outputs[0]) - mul_fold = math_ops.multiply(weights, bn_mult, name=scope + '/mul_fold') - fc_fold = math_ops.matmul(inputs, mul_fold, name=scope + '/MatMul_Fold') - bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub') - .outputs[0]) - add_fold = math_ops.add(fc_fold, bn_bias, name=scope + '/add_fold') + node = fully_connected( + inputs, + out_depth, + weights_initializer=self._WeightInit(0.03), + activation_fn=None, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams(fused_batch_norm), + scope=scope) + # Manually add a bypass (optionaly) and an activation. if with_bypass: - node = math_ops.add(inputs, add_fold, name='test/Add') - else: - node = add_fold + node = math_ops.add(inputs, node, name='test/Add') + node = activation(node, name='test/' + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') + fold_batch_norms.FoldBatchNorms(graph) + quantize.Quantize( graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema) @@ -530,7 +553,8 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def _TestQuantize_DepthwiseConv2dWithBatchNorm( - self, activation, activation_op_name, with_bypass, delay): + self, activation, activation_op_name, with_bypass, delay, + fused_batch_norm): """Tests quantization: inputs -> DWConv2d with batch norm -> Activation. Args: @@ -540,26 +564,30 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. """ self._testQuantize_DepthwiseConv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=True) self._testQuantize_DepthwiseConv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=False) def testQuantize_DepthwiseConv2dWithBatchNorm(self): - self._RunTestOverParameters( - self._TestQuantize_DepthwiseConv2dWithoutBatchNorm) + self._RunBatchNormTestOverParameters( + self._TestQuantize_DepthwiseConv2dWithBatchNorm) def _testQuantize_DepthwiseConv2dWithBatchNorm( - self, activation, activation_op_name, with_bypass, delay, use_ema): + self, activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema): """Tests quantization: inputs -> DWConv2d with batch norm -> Activation. Args: @@ -569,6 +597,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. use_ema: Bool, when true uses EMA quantization for BN folded weights. """ graph = ops.Graph() @@ -579,46 +608,30 @@ class QuantizeTest(test_util.TensorFlowTestCase): inputs = array_ops.zeros((batch_size, height, width, depth)) stride = 1 if with_bypass else 2 scope = 'test/test2' if with_bypass else 'test' - node = separable_conv2d(inputs, None, [5, 5], stride=stride, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) - # Manually fold the batch norm. - weights = (graph.get_operation_by_name(scope + '/depthwise_weights/read') - .outputs[0]) - bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul') - .outputs[0]) - new_shape = [ - weights.get_shape().as_list()[2], weights.get_shape().as_list()[3] - ] - bn_mult_reshaped = array_ops.reshape( - bn_mult, new_shape, name=scope + '/gamma_reshape') - mul_fold = math_ops.multiply( - weights, bn_mult_reshaped, name=scope + '/mul_fold') - stride = [1, stride, stride, 1] - conv_fold = nn_ops.depthwise_conv2d( - input=inputs, - filter=mul_fold, + node = separable_conv2d( + inputs, + None, [5, 5], + stride=stride, + depth_multiplier=1.0, padding='SAME', - strides=stride, - name=scope + '/depthwise_Fold') - bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub') - .outputs[0]) - add_fold = math_ops.add(conv_fold, bn_bias, name=scope + '/add_fold') + weights_initializer=self._WeightInit(0.09), + activation_fn=None, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams(fused_batch_norm), + scope=scope) + # Manually add a bypass (optionaly) and an activation. if with_bypass: - node = math_ops.add(inputs, add_fold, name='test/Add') - else: - node = add_fold + node = math_ops.add(inputs, node, name='test/Add') + node = activation(node, name='test/' + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') + fold_batch_norms.FoldBatchNorms(graph) + quantize.Quantize( graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema) quantization_node_name = 'FakeQuantWithMinMaxVars' @@ -660,6 +673,9 @@ class QuantizeTest(test_util.TensorFlowTestCase): if delay else 'control_dependency') self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) + def _BatchNormParams(self, fused=False): + return {'center': True, 'scale': True, 'decay': 1.0 - 0.003, 'fused': fused} + def _WeightInit(self, stddev): """Returns truncated normal variable initializer. -- GitLab From 49f9c6f890c938955fa2d448ac5b556b9a6d9aa0 Mon Sep 17 00:00:00 2001 From: powderluv Date: Thu, 19 Oct 2017 23:07:55 -0700 Subject: [PATCH 195/573] Fix ../makefile/download_dependencies.sh on OSX (#13845) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wget expects parameters before the URL on OSX (tested on version 1.16 and 1.19) It would fail trying to use -P as a URL Resolving -p... failed: nodename nor servname provided, or not known. wget: unable to resolve host address ‘-p’ --- tensorflow/contrib/makefile/download_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh index 39c89628d9..a63cd89e89 100755 --- a/tensorflow/contrib/makefile/download_dependencies.sh +++ b/tensorflow/contrib/makefile/download_dependencies.sh @@ -54,7 +54,7 @@ download_and_extract() { elif [[ "${url}" == *zip ]]; then tempdir=$(mktemp -d) tempdir2=$(mktemp -d) - wget ${url} -P ${tempdir} + wget -P ${tempdir} ${url} unzip ${tempdir}/* -d ${tempdir2} # unzip has no strip components, so unzip to a temp dir, and move the files # we want from the tempdir to destination. -- GitLab From a528ccdbfe6e4dadad4d982099e8ea5be93fe96f Mon Sep 17 00:00:00 2001 From: Jinze Bai Date: Fri, 20 Oct 2017 23:20:02 +0800 Subject: [PATCH 196/573] Add GPU support and improve performance for tf.diag and tf.diag_part (#13666) * improve tf.diag and tf.diag_part in CPU and GPU * add comment * make changes of DiagOp according to reviews * tidy indent * remove uesless comment prefix * add shard function for DiagOp * add benchmark for diag_op_test in core/kernel * change symbol order in BUILD file * remove empty line for Sanity Checks * add some comments and fix benchmark throughput ratio for DiagOp --- tensorflow/core/graph/testlib.cc | 18 ++ tensorflow/core/graph/testlib.h | 6 + tensorflow/core/kernels/BUILD | 18 ++ tensorflow/core/kernels/diag_op.cc | 295 +++++++++++------- tensorflow/core/kernels/diag_op.h | 43 +++ tensorflow/core/kernels/diag_op_gpu.cu.cc | 150 +++++++++ tensorflow/core/kernels/diag_op_test.cc | 54 ++++ tensorflow/core/ops/array_ops.cc | 10 +- tensorflow/core/ops/array_ops_test.cc | 13 +- .../python/kernel_tests/diag_op_test.py | 64 +++- 10 files changed, 538 insertions(+), 133 deletions(-) create mode 100644 tensorflow/core/kernels/diag_op.h create mode 100644 tensorflow/core/kernels/diag_op_gpu.cu.cc create mode 100644 tensorflow/core/kernels/diag_op_test.cc diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc index be52438747..172471e34b 100644 --- a/tensorflow/core/graph/testlib.cc +++ b/tensorflow/core/graph/testlib.cc @@ -480,6 +480,24 @@ Node* Conv2D(Graph* g, Node* in0, Node* in1) { return ret; } +Node* Diag(Graph* g, Node* in, DataType type) { + Node* ret; + TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Diag") + .Input(in) + .Attr("T", type) + .Finalize(g, &ret)); + return ret; +} + +Node* DiagPart(Graph* g, Node* in, DataType type) { + Node* ret; + TF_CHECK_OK(NodeBuilder(g->NewName("n"), "DiagPart") + .Input(in) + .Attr("T", type) + .Finalize(g, &ret)); + return ret; +} + void ToGraphDef(Graph* g, GraphDef* gdef) { g->ToGraphDef(gdef); } } // end namespace graph diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h index a38809e6b4..06597778bb 100644 --- a/tensorflow/core/graph/testlib.h +++ b/tensorflow/core/graph/testlib.h @@ -199,6 +199,12 @@ Node* BiasAdd(Graph* g, Node* value, Node* bias); // Add a Conv2D node in "g". Node* Conv2D(Graph* g, Node* in0, Node* in1); +// Add a Diag node in "g". +Node* Diag(Graph* g, Node* in, DataType type); + +// Add a DiagPart node in "g". +Node* DiagPart(Graph* g, Node* in, DataType type); + } // end namespace graph } // end namespace test } // end namespace tensorflow diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 3a06189d72..f5700346fd 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2912,6 +2912,24 @@ tf_cuda_cc_test( ], ) +tf_cuda_cc_test( + name = "diag_op_test", + size = "small", + srcs = ["diag_op_test.cc"], + deps = [ + ":diag_op", + ":ops_testutil", + ":ops_util", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + # conv_grad_ops currently has to be built with conv_ops*. # TODO(josh11b, zhengxq): put these a separate libraries in ":nn" below once # conv_ops_gpu.h has be separated into its own library. diff --git a/tensorflow/core/kernels/diag_op.cc b/tensorflow/core/kernels/diag_op.cc index c800859d90..be862b82f1 100644 --- a/tensorflow/core/kernels/diag_op.cc +++ b/tensorflow/core/kernels/diag_op.cc @@ -14,65 +14,32 @@ limitations under the License. ==============================================================================*/ // See docs in ../ops/array_ops.cc + +#define EIGEN_USE_THREADS + +#if GOOGLE_CUDA +#define EIGEN_USE_GPU +#endif // GOOGLE_CUDA + +#include "tensorflow/core/kernels/diag_op.h" + +#include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/work_sharder.h" namespace tensorflow { -namespace { -template -class DiagonalGenerator { - public: - explicit DiagonalGenerator(const Tensor& diagonal) : diagonal_(diagonal) { - static_assert(DoubleNumDims == 2 * NumDims, - "The second size must be the double of the first size."); - CHECK_EQ(diagonal.dims(), NumDims); - } - T operator()( - const Eigen::array& coordinates) const { - Eigen::array index; - for (size_t i = 0; i < NumDims; ++i) { - if (coordinates[i] != coordinates[NumDims + i]) { - return T(0); - } - index[i] = coordinates[i]; - } - return diagonal_.tensor()(index); - } - private: - Tensor diagonal_; -}; - -template -class DiagonalExtractor { - public: - explicit DiagonalExtractor(const Tensor& tensor) : tensor_(tensor) { - CHECK_EQ(tensor.dims(), 2 * NumDims); - } - T operator()(const Eigen::array& coordinates) const { - Eigen::array index; - for (size_t j = 0; j < NumDims; ++j){ - index[j] = coordinates[j]; - } - for (size_t j = NumDims; j < 2 * NumDims; ++j){ - index[j] = index[j - NumDims]; - } - return tensor_.tensor()(index); - } - - private: - Tensor tensor_; -}; - -} // namespace +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef Eigen::GpuDevice GPUDevice; // Generate the diagonal tensor with the diagonal set to the input tensor. -// It only allows up to rank 3 input tensor, so the output tensor is up to -// rank 6. -template +template class DiagOp : public OpKernel { public: explicit DiagOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -80,9 +47,8 @@ class DiagOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& diagonal = context->input(0); const int num_dims = diagonal.dims(); - OP_REQUIRES(context, 1 <= num_dims && num_dims <= 3, - errors::InvalidArgument("Expected 1 <= dims <= 3, got shape ", - diagonal.shape().DebugString())); + OP_REQUIRES(context, 0 != num_dims, errors::InvalidArgument( + "Input must be at least rank 1, got 0")); TensorShape out_shape; for (int i = 0; i < num_dims; ++i) { out_shape.AddDim(diagonal.dim_size(i)); @@ -93,45 +59,17 @@ class DiagOp : public OpKernel { Tensor* output_tensor = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output_tensor)); - switch (num_dims) { - case 1: - output_tensor->tensor() = output_tensor->tensor().generate( - DiagonalGenerator(diagonal)); - break; - case 2: - output_tensor->tensor() = output_tensor->tensor().generate( - DiagonalGenerator(diagonal)); - break; - case 3: - output_tensor->tensor() = output_tensor->tensor().generate( - DiagonalGenerator(diagonal)); - break; - default: - context->SetStatus(errors::Unimplemented( - "Diagonal of rank ", num_dims, " tensor is not supported yet.")); - return; - } + functor::DiagFunctor diagFunc; + Status s = diagFunc(context, + diagonal.NumElements(), + diagonal.flat().data(), + output_tensor->flat().data()); + OP_REQUIRES_OK(context, s); } }; -#define REGISTER_DIAGOP(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("Diag").Device(DEVICE_CPU).TypeConstraint("T"), DiagOp) - -REGISTER_DIAGOP(double); -REGISTER_DIAGOP(float); -REGISTER_DIAGOP(int32); -REGISTER_DIAGOP(int64); -REGISTER_DIAGOP(complex64); -REGISTER_DIAGOP(complex128); - -#undef REGISTER_DIAGOP - - -// Generate the diagonal tensor with the diagonal set to the input tensor. -// It only allows rank 2, 4, or 6 input tensor, so the output tensor is -// rank 1, 2, or 3. -template +// Extract the diagonal tensor with the diagonal set to the input tensor. +template class DiagPartOp : public OpKernel { public: explicit DiagPartOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -140,9 +78,9 @@ class DiagPartOp : public OpKernel { const Tensor& tensor = context->input(0); const int num_dims = tensor.dims(); const int out_dims = num_dims / 2; - OP_REQUIRES(context, 2 == num_dims || 4 == num_dims || 6 == num_dims, - errors::InvalidArgument("The rank of the tensor should be 2, \ - 4, or 6, got shape ", + OP_REQUIRES(context, 0 == num_dims % 2, + errors::InvalidArgument("The rank of the tensor should be \ + even and positive, got shape ", tensor.shape().DebugString())); for (int i = 0; i < out_dims; i++){ OP_REQUIRES(context, tensor.dim_size(i) == tensor.dim_size(i + out_dims), @@ -160,39 +98,158 @@ class DiagPartOp : public OpKernel { Tensor* output = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output)); + functor::DiagPartFunctor diagPartFunc; + Status s = diagPartFunc(context, + out_shape.num_elements(), + tensor.flat().data(), + output->flat().data()); + OP_REQUIRES_OK(context, s); + } +}; - switch (num_dims) { - case 2: - output->tensor() = output->tensor().generate( - DiagonalExtractor(tensor)); - break; - case 4: - output->tensor() = output->tensor().generate( - DiagonalExtractor(tensor)); - break; - case 6: - output->tensor() = output->tensor().generate( - DiagonalExtractor(tensor)); - break; - default: - context->SetStatus(errors::Unimplemented( - "Diagonal of rank ", num_dims, " tensor is not supported yet.")); - return; - } +// Implementation of the functor specialization for CPU. +// +// According to the diagonal definition, +// `output[i1,..., ik, i1,..., ik] = input[i1,..., ik]`, +// +// Let the rank of input is [s1,..., sk], then any offset of input's +// pointer can be represent by coordinate [i1,..., ik], +// where `index = i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik` +// +// Let new_index is the offset of output's pointer with coordinate +// [i1,..., ik, i1,..., ik], then we have +// `new_index = i1*(s2*...sk*s1*...*sk) + i2*(s3*...*sk*s1*...*sk) +... + \ +// ik*(s1*...*sk) + i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik +// = (i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik) * (1 + s1*...*sk) +// = index * (1 + s1*...*sk) +// +// Let `size = s1*...*sk`, we finally have `new_index = index * (1 + size)`, +// which is the transfer function we use below. +// This trick make our implementations clear and easy to be parallel. +namespace functor { +template +struct DiagFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // This subprocess is responsible for writing values in index range + // [start*size, limit*size) + auto subDiag = [in, out, size](int64 start, int64 limit) { + std::fill(out + size * start, out + size * limit, T()); + for (int64 index = start; index < limit; ++index) { + out[(1 + size) * index] = in[index]; + } + }; + + // Here, 5 is a empirical factor of cost_per_unit. + auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); + Shard(worker_threads.num_threads, worker_threads.workers, size, + 5 * size, subDiag); + return Status::OK(); + } +}; + +template +struct DiagPartFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // This subprocess is responsible for extracting values in index range + // [start, limit) + auto subDiagPart = [in, out, size](int64 start, int64 limit) { + for (int64 index = start; index < limit; ++index) { + out[index] = in[(1 + size) * index]; + } + }; + + // Here, 5 is a empirical factor of cost_per_unit. + auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); + Shard(worker_threads.num_threads, worker_threads.workers, size, + 5, subDiagPart); + return Status::OK(); } }; +} // namespace functor -#define REGISTER_DIAGPARTOP(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("DiagPart").Device(DEVICE_CPU).TypeConstraint("T"), DiagPartOp) -REGISTER_DIAGPARTOP(double); -REGISTER_DIAGPARTOP(float); -REGISTER_DIAGPARTOP(int32); -REGISTER_DIAGPARTOP(int64); -REGISTER_DIAGPARTOP(complex64); -REGISTER_DIAGPARTOP(complex128); +// Register the CPU kernels. +#define REGISTER_DIAGOP(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("Diag").Device(DEVICE_CPU).TypeConstraint("T"), \ + DiagOp) +TF_CALL_double(REGISTER_DIAGOP); +TF_CALL_float(REGISTER_DIAGOP); +TF_CALL_int32(REGISTER_DIAGOP); +TF_CALL_int64(REGISTER_DIAGOP); +TF_CALL_complex64(REGISTER_DIAGOP); +TF_CALL_complex128(REGISTER_DIAGOP); +#undef REGISTER_DIAGOP + +#define REGISTER_DIAGPARTOP(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("DiagPart").Device(DEVICE_CPU).TypeConstraint("T"), \ + DiagPartOp) + +TF_CALL_double(REGISTER_DIAGPARTOP); +TF_CALL_float(REGISTER_DIAGPARTOP); +TF_CALL_int32(REGISTER_DIAGPARTOP); +TF_CALL_int64(REGISTER_DIAGPARTOP); +TF_CALL_complex64(REGISTER_DIAGPARTOP); +TF_CALL_complex128(REGISTER_DIAGPARTOP); #undef REGISTER_DIAGPARTOP - + +// Register the GPU kernels. +#ifdef GOOGLE_CUDA + +// Forward declarations of the functor specializations for GPU. +namespace functor { +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +} // namespace functor + +#define REGISTER_DIAGOP_GPU(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("Diag").Device(DEVICE_GPU).TypeConstraint("T"), \ + DiagOp) + +TF_CALL_double(REGISTER_DIAGOP_GPU); +TF_CALL_float(REGISTER_DIAGOP_GPU); +TF_CALL_int32(REGISTER_DIAGOP_GPU); +TF_CALL_int64(REGISTER_DIAGOP_GPU); +TF_CALL_complex64(REGISTER_DIAGOP_GPU); +TF_CALL_complex128(REGISTER_DIAGOP_GPU); +#undef REGISTER_DIAGOP_GPU + +// Forward declarations of the functor specializations for GPU. +namespace functor { +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +} // namespace functor + +#define REGISTER_DIAGPARTOP_GPU(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("DiagPart").Device(DEVICE_GPU).TypeConstraint("T"), \ + DiagPartOp) + +TF_CALL_double(REGISTER_DIAGPARTOP_GPU); +TF_CALL_float(REGISTER_DIAGPARTOP_GPU); +TF_CALL_int32(REGISTER_DIAGPARTOP_GPU); +TF_CALL_int64(REGISTER_DIAGPARTOP_GPU); +TF_CALL_complex64(REGISTER_DIAGPARTOP_GPU); +TF_CALL_complex128(REGISTER_DIAGPARTOP_GPU); +#undef REGISTER_DIAGPARTOP_GPU + +#endif // GOOGLE_CUDA + + } // namespace tensorflow + diff --git a/tensorflow/core/kernels/diag_op.h b/tensorflow/core/kernels/diag_op.h new file mode 100644 index 0000000000..c6ca6a2047 --- /dev/null +++ b/tensorflow/core/kernels/diag_op.h @@ -0,0 +1,43 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_ +#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +namespace functor { + +template +struct DiagFunctor { + Status operator() (OpKernelContext* context, const int64 size, + const T* in, T* out); +}; + +template +struct DiagPartFunctor { + Status operator() (OpKernelContext* context, const int64 size, + const T* in, T* out); +}; + +} // namespace functor + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_ diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc new file mode 100644 index 0000000000..9878f347d2 --- /dev/null +++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc @@ -0,0 +1,150 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" +#include "tensorflow/core/kernels/diag_op.h" + +namespace tensorflow { +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; + +template +__global__ void DiagCudaKernel(const int num_threads, + const int64 size, + const T* in, + T* out) { + CUDA_1D_KERNEL_LOOP(index, num_threads) { + out[(1 + size) * index] = in[index]; + } +} + +template +__global__ void ZeroCudaKernel(const int num_threads, + T* out) { + CUDA_1D_KERNEL_LOOP(index, num_threads) { + out[index] = T(0); + } +} + +template +struct DiagFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // CudaLaunchConfig uses an int for virtual_thread_count, + // so this may overflow in extreme cases. + if (size && (size * size / size) != size) { + return errors::Internal( + "DiagOp got input size too large."); + } + + // Empty tensor couldn't launch the kernel. + if (size == 0) { + return Status::OK(); + } + const GPUDevice& device = context->eigen_device(); + + // Set output memory with zero elements. + CudaLaunchConfig zero_config = GetCudaLaunchConfig(size*size, device); + ZeroCudaKernel<<>>( + zero_config.virtual_thread_count, out); + auto err = cudaGetLastError(); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch DiagOp kernel: ", + cudaGetErrorString(err), "."); + } + + // Fill the diagonal elements + CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device); + DiagCudaKernel<<>>( + diag_config.virtual_thread_count, size, in, out); + err = cudaGetLastError(); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch DiagOp kernel: ", + cudaGetErrorString(err), "."); + } + return Status::OK(); + } +}; + +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; + + +template +__global__ void DiagPartCudaKernel(const int num_threads, + const int64 size, + const T* in, + T* out) { + CUDA_1D_KERNEL_LOOP(index, num_threads) { + out[index] = in[(1 + size) * index]; + } +} + +template +struct DiagPartFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // Empty tensor couldn't launch the kernel. + if (size == 0) { + return Status::OK(); + } + const GPUDevice& device = context->eigen_device(); + + // Extract the diagonal elements. + CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device); + DiagPartCudaKernel<<>>( + diag_config.virtual_thread_count, size, in, out); + auto err = cudaGetLastError(); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch DiagPartOp kernel: ", + cudaGetErrorString(err), "."); + } + return Status::OK(); + } +}; + +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; + +} // end namespace functor +} // end namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/diag_op_test.cc b/tensorflow/core/kernels/diag_op_test.cc new file mode 100644 index 0000000000..2d1417854c --- /dev/null +++ b/tensorflow/core/kernels/diag_op_test.cc @@ -0,0 +1,54 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/test_benchmark.h" + +namespace tensorflow { + +template +static Graph* Diag(int n, DataType type) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor in(type, TensorShape({n})); + in.flat().setRandom(); + Node* out = test::graph::Diag(g, test::graph::Constant(g, in), type); + test::graph::DiagPart(g, out, type); + return g; +} + +#define BM_DiagDev(N, T, TFTYPE, DEVICE) \ + static void BM_Diag##_##N##_##TFTYPE##_##DEVICE(int iters) { \ + testing::UseRealTime(); \ + testing::ItemsProcessed(static_cast(iters) * N * N); \ + test::Benchmark(#DEVICE, Diag(N, TFTYPE)).Run(iters); \ + } \ + BENCHMARK(BM_Diag##_##N##_##TFTYPE##_##DEVICE); + +#define BM_Diag(N) \ + BM_DiagDev(N, int, DT_INT32, cpu); \ + BM_DiagDev(N, float, DT_FLOAT, cpu); \ + BM_DiagDev(N, std::complex, DT_COMPLEX64, cpu); \ + BM_DiagDev(N, int, DT_INT32, gpu); \ + BM_DiagDev(N, float, DT_FLOAT, gpu); \ + BM_DiagDev(N, std::complex, DT_COMPLEX64, gpu); + +BM_Diag(16); +BM_Diag(128); +BM_Diag(512); + +} // end namespace tensorflow + diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 15b09c2c16..c5935141f8 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -739,7 +739,7 @@ REGISTER_OP("Diag") .Attr("T: {float, double, int32, int64, complex64, complex128}") .SetShapeFn([](InferenceContext* c) { ShapeHandle in = c->input(0); - TF_RETURN_IF_ERROR(c->WithRankAtMost(in, 3, &in)); + TF_RETURN_IF_ERROR(c->WithRankAtLeast(in, 1, &in)); // Output shape is original concatenated with itself. ShapeHandle out; TF_RETURN_IF_ERROR(c->Concatenate(in, in, &out)); @@ -767,7 +767,7 @@ tf.diag(diagonal) ==> [[1, 0, 0, 0] [0, 0, 0, 4]] ``` -diagonal: Rank k tensor where k is at most 3. +diagonal: Rank k tensor where k is at most 1. )doc"); // -------------------------------------------------------------------------- @@ -783,9 +783,9 @@ REGISTER_OP("DiagPart") } // Rank must be even, and result will have rank . const int32 rank = c->Rank(in); - if ((rank % 2) != 0 || rank > 6) { + if ((rank % 2) != 0 || rank <= 0) { return errors::InvalidArgument( - "Input must have even rank <= 6, input rank is ", rank); + "Input must have even and non-zero rank, input rank is ", rank); } const int32 mid = rank / 2; @@ -820,7 +820,7 @@ For example: tf.diag_part(input) ==> [1, 2, 3, 4] ``` -input: Rank k tensor where k is 2, 4, or 6. +input: Rank k tensor where k is even and not zero. diagonal: The extracted diagonal. )doc"); diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index a5d7a32e05..94eb120175 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -186,21 +186,20 @@ TEST(ArrayOpsTest, Identity_ShapeFnHandles) { TEST(ArrayOpsTest, Diag_ShapeFn) { ShapeInferenceTestOp op("Diag"); INFER_OK(op, "?", "?"); - INFER_OK(op, "[]", "[]"); INFER_OK(op, "[1,?,3]", "[d0_0,d0_1,d0_2,d0_0,d0_1,d0_2]"); - INFER_ERROR("Shape must be at most rank 3 but is rank 4", op, "[?,1,2,3]"); + INFER_OK(op, "[?,1,2,3]", "[d0_0,d0_1,d0_2,d0_3,d0_0,d0_1,d0_2,d0_3]"); + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[]"); } TEST(ArrayOpsTest, DiagPart_ShapeFn) { ShapeInferenceTestOp op("DiagPart"); INFER_OK(op, "?", "?"); - INFER_OK(op, "[]", "[]"); INFER_OK(op, "[1,?,?,4]", "[d0_0,d0_3]"); INFER_OK(op, "[1,?,3,?,4,3]", "[d0_0,d0_4,d0_2|d0_5]"); - INFER_ERROR("Input must have even rank <= 6, input rank is 1", op, "[?]"); - INFER_ERROR("Input must have even rank <= 6, input rank is 3", op, "[1,2,3]"); - INFER_ERROR("Input must have even rank <= 6, input rank is 8", op, - "[1,2,3,?,?,?,?,?]"); + INFER_OK(op, "[1,2,3,?,?,?,?,4]", "[d0_0,d0_1,d0_2,d0_7]"); + INFER_ERROR("Input must have even and non-zero rank", op, "[]"); + INFER_ERROR("Input must have even and non-zero rank", op, "[?]"); + INFER_ERROR("Input must have even and non-zero rank", op, "[1,2,3]"); INFER_ERROR("Dimensions must be equal, but are 2 and 10", op, "[1,2,?,10]"); } diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py index f0b7885732..6cfa9b37fe 100644 --- a/tensorflow/python/kernel_tests/diag_op_test.py +++ b/tensorflow/python/kernel_tests/diag_op_test.py @@ -279,7 +279,7 @@ class MatrixDiagPartTest(test.TestCase): class DiagTest(test.TestCase): - def diagOp(self, diag, dtype, expected_ans, use_gpu=False): + def _diagOp(self, diag, dtype, expected_ans, use_gpu): with self.test_session(use_gpu=use_gpu): tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype))) out = tf_ans.eval() @@ -290,6 +290,10 @@ class DiagTest(test.TestCase): self.assertShapeEqual(expected_ans, tf_ans) self.assertShapeEqual(diag, tf_ans_inv) + def diagOp(self, diag, dtype, expected_ans): + self._diagOp(diag, dtype, expected_ans, False) + self._diagOp(diag, dtype, expected_ans, True) + def testEmptyTensor(self): x = np.array([]) expected_ans = np.empty([0, 0]) @@ -400,13 +404,53 @@ class DiagTest(test.TestCase): dtype=dtype) self.diagOp(x, dtype, expected_ans) + def testRankFourNumberTensor(self): + for dtype in [np.float32, np.float64, np.int64, np.int32]: + # Input with shape [2, 1, 2, 3] + x = np.array([[[[ 1, 2, 3], + [ 4, 5, 6]]], + [[[ 7, 8, 9], + [10, 11, 12]]]], dtype=dtype) + # Output with shape [2, 1, 2, 3, 2, 1, 2, 3] + expected_ans = np.array( + [[[[[[[[1, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 2, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 3], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]]], + [[[[[0, 0, 0], [4, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 5, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 6]]], + [[[0, 0, 0], [0, 0, 0]]]]]]], + + [[[[[[[0, 0, 0], [0, 0, 0]]], + [[[7, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 8, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 9], [0, 0, 0]]]]], + [[[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [10, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 11, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 12]]]]]]]], dtype=dtype) + self.diagOp(x, dtype, expected_ans) + + def testInvalidRank(self): + with self.assertRaisesRegexp(ValueError, "must be at least rank 1"): + array_ops.diag(0.0) + class DiagPartOpTest(test.TestCase): def setUp(self): np.random.seed(0) - def diagPartOp(self, tensor, dtype, expected_ans, use_gpu=False): + def _diagPartOp(self, tensor, dtype, expected_ans, use_gpu): with self.test_session(use_gpu=use_gpu): tensor = ops.convert_to_tensor(tensor.astype(dtype)) tf_ans_inv = array_ops.diag_part(tensor) @@ -414,6 +458,10 @@ class DiagPartOpTest(test.TestCase): self.assertAllClose(inv_out, expected_ans) self.assertShapeEqual(expected_ans, tf_ans_inv) + def diagPartOp(self, tensor, dtype, expected_ans): + self._diagPartOp(tensor, dtype, expected_ans, False) + self._diagPartOp(tensor, dtype, expected_ans, True) + def testRankTwoFloatTensor(self): x = np.random.rand(3, 3) i = np.arange(3) @@ -451,11 +499,23 @@ class DiagPartOpTest(test.TestCase): self.diagPartOp(x, np.float32, expected_ans) self.diagPartOp(x, np.float64, expected_ans) + def testRankEightComplexTensor(self): + x = np.random.rand(2, 2, 2, 3, 2, 2, 2, 3) + i = np.arange(2)[:, None, None, None] + j = np.arange(2)[:, None, None] + k = np.arange(2)[:, None] + l = np.arange(3) + expected_ans = x[i, j, k, l, i, j, k, l] + self.diagPartOp(x, np.complex64, expected_ans) + self.diagPartOp(x, np.complex128, expected_ans) + def testOddRank(self): w = np.random.rand(2) x = np.random.rand(2, 2, 2) self.assertRaises(ValueError, self.diagPartOp, w, np.float32, 0) self.assertRaises(ValueError, self.diagPartOp, x, np.float32, 0) + with self.assertRaises(ValueError): + array_ops.diag_part(0.0) def testUnevenDimensions(self): w = np.random.rand(2, 5) -- GitLab From 93871a811eab7457f8e36ee4905234aa1a9ea8c8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 09:13:44 -0700 Subject: [PATCH 197/573] Remove duplicated `smart_cond()` code. PiperOrigin-RevId: 172891249 --- .../training/python/training/bucket_ops.py | 4 +-- tensorflow/python/BUILD | 1 + tensorflow/python/training/input.py | 26 ++++--------------- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/tensorflow/contrib/training/python/training/bucket_ops.py b/tensorflow/contrib/training/python/training/bucket_ops.py index 5523cc375f..95fbc50cba 100644 --- a/tensorflow/contrib/training/python/training/bucket_ops.py +++ b/tensorflow/contrib/training/python/training/bucket_ops.py @@ -31,6 +31,7 @@ from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util +from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops @@ -47,7 +48,6 @@ _dtypes = input_py._dtypes _store_sparse_tensors = input_py._store_sparse_tensors _validate_keep_input = input_py._validate_keep_input _shapes = input_py._shapes -_smart_cond = input_py._smart_cond _which_queue = input_py._which_queue # pylint: enable=protected-access @@ -239,7 +239,7 @@ def bucket(tensors, ] return control_flow_ops.group(*enqueues, name="group_enqueues") - maybe_enqueue = _smart_cond( + maybe_enqueue = utils.smart_cond( keep_input, enqueue_which, control_flow_ops.no_op) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 21cdaec477..e63c554e47 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2638,6 +2638,7 @@ py_library( ":init_ops", ":io_ops", ":io_ops_gen", + ":layers_base", ":lib", ":lookup_ops", ":math_ops", diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index 704017c244..36f97960dd 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -32,7 +32,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util +from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops @@ -413,22 +413,6 @@ def _as_original_type(original_tensors, tensor_list): return tensor_list -def _smart_cond(pred, if_true, if_false): - """A `tf.cond` that does nothing when the condition is static.""" - pred = ops.convert_to_tensor(pred) - static_pred = tensor_util.constant_value(pred) - if static_pred is not None: - if static_pred: - return if_true() - else: - return if_false() - else: - return control_flow_ops.cond( - pred, - if_true, - if_false) - - def _store_sparse_tensors(tensor_list, enqueue_many, keep_input, shared_map_ops=None): """Store SparseTensors for feeding into batch, etc. @@ -480,13 +464,13 @@ def _store_sparse_tensors(tensor_list, enqueue_many, keep_input, map_op_name = shared_map_op.name if shared_map_op else None def _maybe_store_sparse(t, map_op_name, keep_input): """Conditionally store a single sparse Tensor.""" - return _smart_cond( + return utils.smart_cond( keep_input, lambda: _store_sparse(t, shared_name=map_op_name), lambda: constant_op.constant(-1, dtypes.int64)) def _maybe_store_many_sparse(t, map_op_name, keep_input): """Conditionally store multiple sparse Tensors.""" - out_tensor = _smart_cond( + out_tensor = utils.smart_cond( keep_input, lambda: _store_many_sparse(t, shared_name=map_op_name), lambda: -1 * array_ops.ones(array_ops.shape(t)[0:1], dtypes.int64)) @@ -667,7 +651,7 @@ def _enqueue_join(queue, tensor_list_list, enqueue_many, keep_input): enqueue_ops = [enqueue_fn(_select_which_to_enqueue(x, keep_input)) for x in tensor_list_list] else: - enqueue_ops = [_smart_cond( + enqueue_ops = [utils.smart_cond( keep_input, lambda: enqueue_fn(tl), # pylint:disable=cell-var-from-loop control_flow_ops.no_op) for tl in tensor_list_list] @@ -684,7 +668,7 @@ def _enqueue(queue, tensor_list, threads, enqueue_many, keep_input): enqueue_ops = [ enqueue_fn(_select_which_to_enqueue(tensor_list, keep_input))] * threads else: - enqueue_ops = [_smart_cond( + enqueue_ops = [utils.smart_cond( keep_input, lambda: enqueue_fn(tensor_list), control_flow_ops.no_op)] * threads -- GitLab From 5c24b8b1e5f3f1145e123a5a159b958ea9fc8c3d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 09:17:03 -0700 Subject: [PATCH 198/573] XLA refactoring PiperOrigin-RevId: 172891551 --- .../xla/legacy_flags/debug_options_flags.cc | 6 ++--- tensorflow/compiler/xla/protobuf_util.cc | 25 ----------------- tensorflow/compiler/xla/protobuf_util.h | 13 +++------ .../compiler/xla/service/cpu/cpu_compiler.cc | 27 +++++++++---------- .../compiler/xla/service/gpu/gpu_compiler.cc | 10 +++---- tensorflow/compiler/xla/xla.proto | 4 +-- 6 files changed, 27 insertions(+), 58 deletions(-) diff --git a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc index 8892bfbe92..f2cdd9669c 100644 --- a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc +++ b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc @@ -206,9 +206,9 @@ void AllocateFlags() { flag_values->xla_gpu_disable_multi_streaming(), "If true, multi-streaming in the GPU backend is disabled."), tensorflow::Flag( - "xla_dump_debug_json_to", - flag_values->mutable_xla_dump_debug_json_to(), - "Dump compilation artifacts as JSON into this directory."), + "xla_dump_hlo_proto_to", + flag_values->mutable_xla_dump_hlo_proto_to(), + "Dump compilation artifacts as proto binary into this directory."), tensorflow::Flag( "xla_test_all_output_layouts", bool_setter_for(&DebugOptions::set_xla_test_all_output_layouts), diff --git a/tensorflow/compiler/xla/protobuf_util.cc b/tensorflow/compiler/xla/protobuf_util.cc index c032cb8dc5..787725e884 100644 --- a/tensorflow/compiler/xla/protobuf_util.cc +++ b/tensorflow/compiler/xla/protobuf_util.cc @@ -37,20 +37,6 @@ bool ProtobufEquals(const tensorflow::protobuf::Message& m1, return (serialized1 == serialized2); } -StatusOr ToJson(const tensorflow::protobuf::Message& message) { - string json_output; - tensorflow::protobuf::util::JsonPrintOptions json_options; - json_options.add_whitespace = true; - json_options.always_print_primitive_fields = true; - auto status = tensorflow::protobuf::util::MessageToJsonString( - message, &json_output, json_options); - if (!status.ok()) { - return InternalError("MessageToJsonString failed: %s", - status.error_message().data()); - } - return json_output; -} - namespace { string SanitizeFilename(const string& file_name) { @@ -65,17 +51,6 @@ string SanitizeFilename(const string& file_name) { } // namespace -Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message, - const string& directory, const string& file_name) { - TF_ASSIGN_OR_RETURN(const string json_output, ToJson(message)); - - tensorflow::Env* env = tensorflow::Env::Default(); - TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory)); - string safe_file_name = SanitizeFileName(file_name) + ".json"; - const string path = tensorflow::io::JoinPath(directory, safe_file_name); - return tensorflow::WriteStringToFile(env, path, json_output); -} - Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message, const string& directory, const string& file_name) { tensorflow::Env* env = tensorflow::Env::Default(); diff --git a/tensorflow/compiler/xla/protobuf_util.h b/tensorflow/compiler/xla/protobuf_util.h index 7accb22e0c..3667621367 100644 --- a/tensorflow/compiler/xla/protobuf_util.h +++ b/tensorflow/compiler/xla/protobuf_util.h @@ -32,17 +32,12 @@ namespace protobuf_util { extern bool ProtobufEquals(const tensorflow::protobuf::Message& m1, const tensorflow::protobuf::Message& m2); -// Returns 'message' as a JSON string. -StatusOr ToJson(const tensorflow::protobuf::Message& message); - -// Writes the given message in binary proto or JSON format to the path formed by -// joining 'directory/file_name.pb' (or file_name.json). The 'directory' is -// recursively created if it doesn't already exist, and the 'file_name' is -// sanitized by replacing illegal characters with underscore '_'. +// Writes the given message in binary proto to the path formed by joining +// 'directory/file_name.pb'. The 'directory' is recursively created if it +// doesn't already exist, and the 'file_name' is sanitized by replacing +// illegal characters with underscore '_'. Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message, const string& directory, const string& file_name); -Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message, - const string& directory, const string& file_name); } // namespace protobuf_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index ce4d109214..06e7ec0c7c 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -475,8 +475,8 @@ StatusOr> CpuCompiler::Compile( // ownership is std::moved. const bool embed_ir_in_executable = module->config().debug_options().xla_embed_ir_in_executable(); - const string dump_debug_json_to = - module->config().debug_options().xla_dump_debug_json_to(); + const string xla_dump_hlo_proto_to = + module->config().debug_options().xla_dump_hlo_proto_to(); if (options::CpuParallelBackendRequested(module->config())) { VLOG(1) << "Using parallel cpu backend"; @@ -496,10 +496,10 @@ StatusOr> CpuCompiler::Compile( // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); - if (!dump_debug_json_to.empty()) { + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } // If we are using the parallel CPU backend, we need to create map from @@ -603,12 +603,11 @@ StatusOr> CpuCompiler::Compile( // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); - if (!dump_debug_json_to.empty()) { + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } - // Each computation is a single function. Emit all embedded computations // before the entry computation. The order of computations returned from // GetEmbeddedComputations guarantees that a called computation occurs @@ -775,12 +774,12 @@ CpuCompiler::CompileAheadOfTime(std::vector> modules, // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); - const string dump_debug_json_to = - module->config().debug_options().xla_dump_debug_json_to(); - if (!dump_debug_json_to.empty()) { + const string xla_dump_hlo_proto_to = + module->config().debug_options().xla_dump_hlo_proto_to(); + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } IrEmitter ir_emitter(*module, *assignment, &llvm_module, diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 3e16e4e3c4..9c7ca9ea38 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -318,12 +318,12 @@ StatusOr> GpuCompiler::Compile( // print one ourselves. XLA_VLOG_LINES(2, buffer_assignment->ToString()); - const string dump_debug_json_to = - module->config().debug_options().xla_dump_debug_json_to(); - if (!dump_debug_json_to.empty()) { + const string xla_dump_hlo_proto_to = + module->config().debug_options().xla_dump_hlo_proto_to(); + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *buffer_assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } IrEmitterContext ir_emitter_context(module.get(), buffer_assignment.get(), diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto index 7f4bd26d1b..ce3c3eee68 100644 --- a/tensorflow/compiler/xla/xla.proto +++ b/tensorflow/compiler/xla/xla.proto @@ -82,8 +82,8 @@ message DebugOptions { // Dump all HLO modules as text into the provided directory path. string xla_generate_hlo_text_to = 7; - // Dump compilation artifacts as JSON into this directory. - string xla_dump_debug_json_to = 8; + // Dump compilation artifacts in binary proto into this directory. + string xla_dump_hlo_proto_to = 8; // Instrument the computation to collect per-HLO cycle counts. bool xla_hlo_profile = 9; -- GitLab From f86588ce8fb38ab3a6afc21eb08d2a2097b56adc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 09:50:28 -0700 Subject: [PATCH 199/573] Added gradient op for QR decomposition PiperOrigin-RevId: 172895297 --- tensorflow/python/kernel_tests/qr_op_test.py | 66 ++++++++++++++++++-- tensorflow/python/ops/linalg_grad.py | 42 +++++++++++-- 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py index b4fd89bd03..8848c15e76 100644 --- a/tensorflow/python/kernel_tests/qr_op_test.py +++ b/tensorflow/python/kernel_tests/qr_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -140,11 +141,11 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_): x_reshape = np.reshape(x_np, (-1, x_np.shape[-2], x_np.shape[-1])) for i in range(new_first_dim): if full_matrices_: - np_q_reshape[i,:,:], _ = \ - np.linalg.qr(x_reshape[i,:,:], mode="complete") + np_q_reshape[i, :, :], _ = np.linalg.qr( + x_reshape[i, :, :], mode="complete") else: - np_q_reshape[i,:,:], _ = \ - np.linalg.qr(x_reshape[i,:,:], mode="reduced") + np_q_reshape[i, :, :], _ = np.linalg.qr( + x_reshape[i, :, :], mode="reduced") np_q = np.reshape(np_q_reshape, q_dims) CompareOrthogonal(self, np_q, q_tf_val, min(shape_[-2:])) CheckApproximation(self, x_np, q_tf_val, r_tf_val) @@ -153,6 +154,46 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_): return Test +class QrGradOpTest(test.TestCase): + pass + + +def _GetQrGradOpTest(dtype_, shape_, full_matrices_): + + def Test(self): + np.random.seed(42) + a = np.random.uniform(low=-1.0, high=1.0, size=shape_).astype(dtype_) + if dtype_ in [np.complex64, np.complex128]: + a += 1j * np.random.uniform( + low=-1.0, high=1.0, size=shape_).astype(dtype_) + # Optimal stepsize for central difference is O(epsilon^{1/3}). + epsilon = np.finfo(dtype_).eps + delta = 0.1 * epsilon**(1.0 / 3.0) + if dtype_ in [np.float32, np.complex64]: + tol = 3e-2 + else: + tol = 1e-6 + with self.test_session(use_gpu=True): + tf_a = constant_op.constant(a) + tf_b = linalg_ops.qr(tf_a, full_matrices=full_matrices_) + for b in tf_b: + x_init = np.random.uniform( + low=-1.0, high=1.0, size=shape_).astype(dtype_) + if dtype_ in [np.complex64, np.complex128]: + x_init += 1j * np.random.uniform( + low=-1.0, high=1.0, size=shape_).astype(dtype_) + theoretical, numerical = gradient_checker.compute_gradient( + tf_a, + tf_a.get_shape().as_list(), + b, + b.get_shape().as_list(), + x_init_value=x_init, + delta=delta) + self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol) + + return Test + + if __name__ == "__main__": for dtype in np.float32, np.float64, np.complex64, np.complex128: for rows in 1, 2, 5, 10, 32, 100: @@ -168,4 +209,21 @@ if __name__ == "__main__": _AddTest(QrOpTest, "Qr", name, _GetQrOpTest(dtype, shape, full_matrices, use_static_shape)) + + # TODO(pfau): Get working with complex types. + # TODO(pfau): Get working with full_matrices when rows != cols + # TODO(pfau): Get working when rows < cols + # TODO(pfau): Get working with shapeholders (dynamic shapes) + for full_matrices in False, True: + for dtype in np.float32, np.float64: + for rows in 1, 2, 5, 10: + for cols in 1, 2, 5, 10: + if rows == cols or (not full_matrices and rows > cols): + for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10): + shape = batch_dims + (rows, cols) + name = "%s_%s_full_%s" % (dtype.__name__, + "_".join(map(str, shape)), + full_matrices) + _AddTest(QrGradOpTest, "QrGrad", name, + _GetQrGradOpTest(dtype, shape, full_matrices)) test.main() diff --git a/tensorflow/python/ops/linalg_grad.py b/tensorflow/python/ops/linalg_grad.py index ec263591e1..8a76fe3ce5 100644 --- a/tensorflow/python/ops/linalg_grad.py +++ b/tensorflow/python/ops/linalg_grad.py @@ -81,6 +81,36 @@ def _CholeskyGrad(op, grad): return grad_a * 0.5 +@ops.RegisterGradient("Qr") +def _QrGrad(op, dq, dr): + """Gradient for Qr.""" + q, r = op.outputs + if q.dtype.is_complex: + raise NotImplementedError("QrGrad not implemented for dtype: %s" % q.dtype) + if (r.shape.ndims is None or r.shape.as_list()[-2] is None or + r.shape.as_list()[-1] is None): + raise NotImplementedError("QrGrad not implemented with dynamic shapes.") + if r.shape[-2].value != r.shape[-1].value: + raise NotImplementedError("QrGrad not implemented when ncols > nrows " + "or full_matrices is true and ncols != nrows.") + + qdq = math_ops.matmul(q, dq, adjoint_a=True) + qdq_ = qdq - _linalg.adjoint(qdq) + rdr = math_ops.matmul(r, dr, adjoint_b=True) + rdr_ = rdr - _linalg.adjoint(rdr) + tril = array_ops.matrix_band_part(qdq_ + rdr_, -1, 0) + + def _TriangularSolve(x, r): + """Equiv to matmul(x, adjoint(matrix_inverse(r))) if r is upper-tri.""" + return _linalg.adjoint( + linalg_ops.matrix_triangular_solve( + r, _linalg.adjoint(x), lower=False, adjoint=False)) + + grad_a = math_ops.matmul(q, dr + _TriangularSolve(tril, r)) + grad_b = _TriangularSolve(dq - math_ops.matmul(q, qdq), r) + return grad_a + grad_b + + @ops.RegisterGradient("MatrixSolve") def _MatrixSolveGrad(op, grad): """Gradient for MatrixSolve.""" @@ -105,7 +135,7 @@ def _MatrixSolveLsGrad(op, grad): # b) Implement a symmetric rank-k update op instead of computing # x*z + transpose(x*z). This pattern occurs other places in TensorFlow. - def _overdetermined(op, grad): + def _Overdetermined(op, grad): """Gradients for the overdetermined case of MatrixSolveLs. This is the backprop for the solution to the normal equations of the first @@ -130,7 +160,7 @@ def _MatrixSolveLsGrad(op, grad): grad_b = math_ops.matmul(a, z) return (grad_a, grad_b, None) - def _underdetermined(op, grad): + def _Underdetermined(op, grad): """Gradients for the underdetermined case of MatrixSolveLs. This is the backprop for the solution to the normal equations of the second @@ -162,16 +192,16 @@ def _MatrixSolveLsGrad(op, grad): matrix_shape = op.inputs[0].get_shape()[-2:] if matrix_shape.is_fully_defined(): if matrix_shape[-2] >= matrix_shape[-1]: - return _overdetermined(op, grad) + return _Overdetermined(op, grad) else: - return _underdetermined(op, grad) + return _Underdetermined(op, grad) else: # We have to defer determining the shape to runtime and use # conditional execution of the appropriate graph. matrix_shape = array_ops.shape(op.inputs[0])[-2:] return control_flow_ops.cond(matrix_shape[-2] >= matrix_shape[-1], - lambda: _overdetermined(op, grad), - lambda: _underdetermined(op, grad)) + lambda: _Overdetermined(op, grad), + lambda: _Underdetermined(op, grad)) @ops.RegisterGradient("MatrixTriangularSolve") -- GitLab From c91dadb3737395de6b09f4f52596d7ce202eff8f Mon Sep 17 00:00:00 2001 From: Max Galkin Date: Fri, 20 Oct 2017 10:43:56 -0700 Subject: [PATCH 200/573] Minor change: extra logging to help understand the effects of OptimizeGraph and PruneGraph calls. PiperOrigin-RevId: 172902338 --- tensorflow/core/grappler/grappler_item_builder.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc index 54d60cd7aa..3f6183b6f1 100644 --- a/tensorflow/core/grappler/grappler_item_builder.cc +++ b/tensorflow/core/grappler/grappler_item_builder.cc @@ -450,12 +450,16 @@ std::unique_ptr GrapplerItemFromMetaGraphDef( } // Optimize the graph (function inlining, l1 optimizations, etc). + VLOG(1) << "Number of nodes in graph before OptimizeGraph: " + << new_item->graph.node_size(); Status optimize_status = OptimizeGraph(new_item->graph, &new_item->graph, cfg); if (!optimize_status.ok()) { LOG(ERROR) << "Graph preprocessing failed: " << optimize_status; return nullptr; } + VLOG(1) << "Number of nodes in graph after OptimizeGraph: " + << new_item->graph.node_size(); if (cfg.prune_graph) { VLOG(1) << "Pruning graph..."; @@ -464,7 +468,8 @@ std::unique_ptr GrapplerItemFromMetaGraphDef( LOG(ERROR) << "Pruning failed: " << status.error_message(); return nullptr; } - VLOG(1) << "Pruning ran succesfully."; + VLOG(1) << "Number of nodes in graph after pruning: " + << new_item->graph.node_size(); } // Validate feed, fetch and init nodes -- GitLab From 8f7439888c7c3ea7f188df64952cfb4f1e082ecc Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Fri, 20 Oct 2017 10:45:35 -0700 Subject: [PATCH 201/573] Patch dynamic_rnn to work in Eager mode PiperOrigin-RevId: 172902635 --- tensorflow/contrib/rnn/BUILD | 2 + .../rnn/python/kernel_tests/core_rnn_test.py | 364 +++++++++++------- tensorflow/python/BUILD | 1 + tensorflow/python/kernel_tests/BUILD | 2 + tensorflow/python/kernel_tests/rnn_test.py | 91 +++-- tensorflow/python/ops/rnn.py | 76 ++-- 6 files changed, 339 insertions(+), 197 deletions(-) diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD index 571d299ad9..29ba26d75d 100644 --- a/tensorflow/contrib/rnn/BUILD +++ b/tensorflow/contrib/rnn/BUILD @@ -156,6 +156,7 @@ cuda_py_tests( "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", @@ -165,6 +166,7 @@ cuda_py_tests( "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", ], shard_count = 10, ) diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py index 2fa033632a..12def6dcc8 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py @@ -25,10 +25,12 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.contrib import rnn as rnn_lib from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops as ops_lib from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl @@ -881,6 +883,7 @@ class LSTMTest(test.TestCase): # Smoke test, this should not raise an error rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) + @test_util.run_in_graph_and_eager_modes() def testDynamicRNNWithTupleStates(self): num_units = 3 input_size = 5 @@ -888,13 +891,20 @@ class LSTMTest(test.TestCase): num_proj = 4 max_length = 8 sequence_length = [4, 6] + in_graph_mode = context.in_graph_mode() with self.test_session(graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, shape=(None, input_size)) - ] + if in_graph_mode: + inputs = max_length * [ + array_ops.placeholder( + dtypes.float32, shape=(None, input_size)) + ] + else: + inputs = max_length * [ + constant_op.constant( + np.random.randn(batch_size, input_size).astype(np.float32)) + ] inputs_c = array_ops.stack(inputs) cell = rnn_cell.LSTMCell( num_units, @@ -924,21 +934,34 @@ class LSTMTest(test.TestCase): self.assertEqual(state_dynamic[0], state_dynamic.c) self.assertEqual(state_dynamic[1], state_dynamic.h) - variables_lib.global_variables_initializer().run() - - input_value = np.random.randn(batch_size, input_size) - outputs_static_v = sess.run(outputs_static, - feed_dict={inputs[0]: input_value}) - outputs_dynamic_v = sess.run(outputs_dynamic, - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(outputs_static_v, outputs_dynamic_v) - - state_static_v = sess.run(state_static, - feed_dict={inputs[0]: input_value}) - state_dynamic_v = sess.run(state_dynamic, - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(np.hstack(state_static_v), np.hstack(state_dynamic_v)) + if in_graph_mode: + variables_lib.global_variables_initializer().run() + input_value = np.random.randn(batch_size, input_size) + outputs_static = sess.run( + outputs_static, feed_dict={ + inputs[0]: input_value + }) + outputs_dynamic = sess.run( + outputs_dynamic, feed_dict={ + inputs[0]: input_value + }) + state_static = sess.run( + state_static, feed_dict={ + inputs[0]: input_value + }) + state_dynamic = sess.run( + state_dynamic, feed_dict={ + inputs[0]: input_value + }) + + if in_graph_mode: + self.assertAllEqual(outputs_static, outputs_dynamic) + else: + self.assertAllEqual( + array_ops.stack(outputs_static).numpy(), outputs_dynamic.numpy()) + self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic)) + @test_util.run_in_graph_and_eager_modes() def testDynamicRNNWithNestedTupleStates(self): num_units = 3 input_size = 5 @@ -946,13 +969,20 @@ class LSTMTest(test.TestCase): num_proj = 4 max_length = 8 sequence_length = [4, 6] + in_graph_mode = context.in_graph_mode() with self.test_session(graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, shape=(None, input_size)) - ] + if in_graph_mode: + inputs = max_length * [ + array_ops.placeholder( + dtypes.float32, shape=(None, input_size)) + ] + else: + inputs = max_length * [ + constant_op.constant( + np.random.randn(batch_size, input_size).astype(np.float32)) + ] inputs_c = array_ops.stack(inputs) def _cell(i): @@ -993,20 +1023,34 @@ class LSTMTest(test.TestCase): sequence_length=sequence_length, scope=scope) - variables_lib.global_variables_initializer().run() - - input_value = np.random.randn(batch_size, input_size) - outputs_static_v = sess.run(outputs_static, - feed_dict={inputs[0]: input_value}) - outputs_dynamic_v = sess.run(outputs_dynamic, - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(outputs_static_v, outputs_dynamic_v) - - state_static_v = sess.run(nest.flatten(state_static), - feed_dict={inputs[0]: input_value}) - state_dynamic_v = sess.run(nest.flatten(state_dynamic), - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(np.hstack(state_static_v), np.hstack(state_dynamic_v)) + if in_graph_mode: + input_value = np.random.randn(batch_size, input_size) + variables_lib.global_variables_initializer().run() + outputs_static = sess.run( + outputs_static, feed_dict={ + inputs[0]: input_value + }) + outputs_dynamic = sess.run( + outputs_dynamic, feed_dict={ + inputs[0]: input_value + }) + state_static = sess.run( + nest.flatten(state_static), feed_dict={ + inputs[0]: input_value + }) + state_dynamic = sess.run( + nest.flatten(state_dynamic), feed_dict={ + inputs[0]: input_value + }) + + if in_graph_mode: + self.assertAllEqual(outputs_static, outputs_dynamic) + else: + self.assertAllEqual( + array_ops.stack(outputs_static).numpy(), outputs_dynamic.numpy()) + state_static = [s.numpy() for s in nest.flatten(state_static)] + state_dynamic = [s.numpy() for s in nest.flatten(state_dynamic)] + self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic)) def _testDynamicEquivalentToStaticRNN(self, use_gpu, use_sequence_length): time_steps = 8 @@ -1015,21 +1059,22 @@ class LSTMTest(test.TestCase): input_size = 5 batch_size = 2 - input_values = np.random.randn(time_steps, batch_size, input_size) + input_values = np.random.randn(time_steps, batch_size, input_size).astype( + np.float32) if use_sequence_length: sequence_length = np.random.randint(0, time_steps, size=batch_size) else: sequence_length = None - ########### Step 1: Run static graph and generate readouts - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: - concat_inputs = array_ops.placeholder( - dtypes.float32, shape=(time_steps, batch_size, input_size)) - inputs = array_ops.unstack(concat_inputs) + in_graph_mode = context.in_graph_mode() + + # TODO(b/68017812): Eager ignores operation seeds, so we need to create a + # single cell and reuse it across the static and dynamic RNNs. Remove this + # special case once is fixed. + if not in_graph_mode: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - cell = rnn_cell.LSTMCell( num_units, use_peepholes=True, @@ -1037,63 +1082,85 @@ class LSTMTest(test.TestCase): num_proj=num_proj, state_is_tuple=False) + ########### Step 1: Run static graph and generate readouts + with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + if in_graph_mode: + concat_inputs = array_ops.placeholder( + dtypes.float32, shape=(time_steps, batch_size, input_size)) + else: + concat_inputs = constant_op.constant(input_values) + inputs = array_ops.unstack(concat_inputs) + initializer = init_ops.random_uniform_initializer( + -0.01, 0.01, seed=self._seed) + + # TODO(akshayka): Remove special case once b/68017812 is fixed. + if in_graph_mode: + cell = rnn_cell.LSTMCell( + num_units, + use_peepholes=True, + initializer=initializer, + num_proj=num_proj, + state_is_tuple=False) + with variable_scope.variable_scope("dynamic_scope"): outputs_static, state_static = rnn.static_rnn( cell, inputs, sequence_length=sequence_length, dtype=dtypes.float32) - feeds = {concat_inputs: input_values} - - # Initialize - variables_lib.global_variables_initializer().run(feed_dict=feeds) - - # Generate gradients of sum of outputs w.r.t. inputs - static_gradients = gradients_impl.gradients( - outputs_static + [state_static], [concat_inputs]) - - # Generate gradients of individual outputs w.r.t. inputs - static_individual_gradients = nest.flatten([ - gradients_impl.gradients(y, [concat_inputs]) - for y in [outputs_static[0], outputs_static[-1], state_static] - ]) - - # Generate gradients of individual variables w.r.t. inputs - trainable_variables = ops_lib.get_collection( - ops_lib.GraphKeys.TRAINABLE_VARIABLES) - assert len(trainable_variables) > 1, ("Count of trainable variables: %d" % - len(trainable_variables)) - # pylint: disable=bad-builtin - static_individual_variable_gradients = nest.flatten([ - gradients_impl.gradients(y, trainable_variables) - for y in [outputs_static[0], outputs_static[-1], state_static] - ]) - - # Test forward pass - values_static = sess.run(outputs_static, feed_dict=feeds) - (state_value_static,) = sess.run((state_static,), feed_dict=feeds) - - # Test gradients to inputs and variables w.r.t. outputs & final state - static_grad_values = sess.run(static_gradients, feed_dict=feeds) - - static_individual_grad_values = sess.run(static_individual_gradients, - feed_dict=feeds) - - static_individual_var_grad_values = sess.run( - static_individual_variable_gradients, feed_dict=feeds) + if in_graph_mode: + # Generate gradients and run sessions to obtain outputs + feeds = {concat_inputs: input_values} + # Initialize + variables_lib.global_variables_initializer().run(feed_dict=feeds) + # Generate gradients of sum of outputs w.r.t. inputs + static_gradients = gradients_impl.gradients( + outputs_static + [state_static], [concat_inputs]) + # Generate gradients of individual outputs w.r.t. inputs + static_individual_gradients = nest.flatten([ + gradients_impl.gradients(y, [concat_inputs]) + for y in [outputs_static[0], outputs_static[-1], state_static] + ]) + # Generate gradients of individual variables w.r.t. inputs + trainable_variables = ops_lib.get_collection( + ops_lib.GraphKeys.TRAINABLE_VARIABLES) + assert len(trainable_variables) > 1, ( + "Count of trainable variables: %d" % len(trainable_variables)) + # pylint: disable=bad-builtin + static_individual_variable_gradients = nest.flatten([ + gradients_impl.gradients(y, trainable_variables) + for y in [outputs_static[0], outputs_static[-1], state_static] + ]) + # Test forward pass + values_static = sess.run(outputs_static, feed_dict=feeds) + (state_value_static,) = sess.run((state_static,), feed_dict=feeds) + + # Test gradients to inputs and variables w.r.t. outputs & final state + static_grad_values = sess.run(static_gradients, feed_dict=feeds) + + static_individual_grad_values = sess.run(static_individual_gradients, + feed_dict=feeds) + + static_individual_var_grad_values = sess.run( + static_individual_variable_gradients, feed_dict=feeds) ########## Step 2: Run dynamic graph and generate readouts with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: - concat_inputs = array_ops.placeholder( - dtypes.float32, shape=(time_steps, batch_size, input_size)) - inputs = array_ops.unstack(concat_inputs) + if in_graph_mode: + concat_inputs = array_ops.placeholder( + dtypes.float32, shape=(time_steps, batch_size, input_size)) + else: + concat_inputs = constant_op.constant(input_values) initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - cell = rnn_cell.LSTMCell( - num_units, - use_peepholes=True, - initializer=initializer, - num_proj=num_proj, - state_is_tuple=False) + # TODO(akshayka): Remove this special case once b/68017812 is + # fixed. + if in_graph_mode: + cell = rnn_cell.LSTMCell( + num_units, + use_peepholes=True, + initializer=initializer, + num_proj=num_proj, + state_is_tuple=False) with variable_scope.variable_scope("dynamic_scope"): outputs_dynamic, state_dynamic = rnn.dynamic_rnn( @@ -1104,72 +1171,83 @@ class LSTMTest(test.TestCase): dtype=dtypes.float32) split_outputs_dynamic = array_ops.unstack(outputs_dynamic, time_steps) - feeds = {concat_inputs: input_values} + if in_graph_mode: + feeds = {concat_inputs: input_values} - # Initialize - variables_lib.global_variables_initializer().run(feed_dict=feeds) + # Initialize + variables_lib.global_variables_initializer().run(feed_dict=feeds) + + # Generate gradients of sum of outputs w.r.t. inputs + dynamic_gradients = gradients_impl.gradients( + split_outputs_dynamic + [state_dynamic], [concat_inputs]) + + # Generate gradients of several individual outputs w.r.t. inputs + dynamic_individual_gradients = nest.flatten([ + gradients_impl.gradients(y, [concat_inputs]) + for y in + [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] + ]) + + # Generate gradients of individual variables w.r.t. inputs + trainable_variables = ops_lib.get_collection( + ops_lib.GraphKeys.TRAINABLE_VARIABLES) + assert len(trainable_variables) > 1, ( + "Count of trainable variables: %d" % len(trainable_variables)) + dynamic_individual_variable_gradients = nest.flatten([ + gradients_impl.gradients(y, trainable_variables) + for y in + [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] + ]) - # Generate gradients of sum of outputs w.r.t. inputs - dynamic_gradients = gradients_impl.gradients( - split_outputs_dynamic + [state_dynamic], [concat_inputs]) - - # Generate gradients of several individual outputs w.r.t. inputs - dynamic_individual_gradients = nest.flatten([ - gradients_impl.gradients(y, [concat_inputs]) - for y in - [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] - ]) - - # Generate gradients of individual variables w.r.t. inputs - trainable_variables = ops_lib.get_collection( - ops_lib.GraphKeys.TRAINABLE_VARIABLES) - assert len(trainable_variables) > 1, ("Count of trainable variables: %d" % - len(trainable_variables)) - dynamic_individual_variable_gradients = nest.flatten([ - gradients_impl.gradients(y, trainable_variables) - for y in - [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] - ]) - - # Test forward pass - values_dynamic = sess.run(split_outputs_dynamic, feed_dict=feeds) - (state_value_dynamic,) = sess.run((state_dynamic,), feed_dict=feeds) - - # Test gradients to inputs and variables w.r.t. outputs & final state - dynamic_grad_values = sess.run(dynamic_gradients, feed_dict=feeds) - - dynamic_individual_grad_values = sess.run(dynamic_individual_gradients, - feed_dict=feeds) - - dynamic_individual_var_grad_values = sess.run( - dynamic_individual_variable_gradients, feed_dict=feeds) + # Test forward pass + values_dynamic = sess.run(split_outputs_dynamic, feed_dict=feeds) + (state_value_dynamic,) = sess.run((state_dynamic,), feed_dict=feeds) + + # Test gradients to inputs and variables w.r.t. outputs & final state + dynamic_grad_values = sess.run(dynamic_gradients, feed_dict=feeds) + + dynamic_individual_grad_values = sess.run(dynamic_individual_gradients, + feed_dict=feeds) + + dynamic_individual_var_grad_values = sess.run( + dynamic_individual_variable_gradients, feed_dict=feeds) ######### Step 3: Comparisons + if not in_graph_mode: + values_static = outputs_static + values_dynamic = split_outputs_dynamic + state_value_static = state_static + state_value_dynamic = state_dynamic + self.assertEqual(len(values_static), len(values_dynamic)) for (value_static, value_dynamic) in zip(values_static, values_dynamic): self.assertAllEqual(value_static, value_dynamic) self.assertAllEqual(state_value_static, state_value_dynamic) - self.assertAllEqual(static_grad_values, dynamic_grad_values) + if in_graph_mode: + + self.assertAllEqual(static_grad_values, dynamic_grad_values) - self.assertEqual( - len(static_individual_grad_values), len(dynamic_individual_grad_values)) - self.assertEqual( - len(static_individual_var_grad_values), - len(dynamic_individual_var_grad_values)) + self.assertEqual( + len(static_individual_grad_values), + len(dynamic_individual_grad_values)) + self.assertEqual( + len(static_individual_var_grad_values), + len(dynamic_individual_var_grad_values)) - for i, (a, b) in enumerate( - zip(static_individual_grad_values, dynamic_individual_grad_values)): - tf_logging.info("Comparing individual gradients iteration %d" % i) - self.assertAllEqual(a, b) + for i, (a, b) in enumerate( + zip(static_individual_grad_values, dynamic_individual_grad_values)): + tf_logging.info("Comparing individual gradients iteration %d" % i) + self.assertAllEqual(a, b) - for i, (a, b) in enumerate( - zip(static_individual_var_grad_values, - dynamic_individual_var_grad_values)): - tf_logging.info("Comparing individual variable gradients iteration %d" % - i) - self.assertAllEqual(a, b) + for i, (a, b) in enumerate( + zip(static_individual_var_grad_values, + dynamic_individual_var_grad_values)): + tf_logging.info("Comparing individual variable gradients iteration %d" % + i) + self.assertAllEqual(a, b) + @test_util.run_in_graph_and_eager_modes() def testDynamicEquivalentToStaticRNN(self): self._testDynamicEquivalentToStaticRNN( use_gpu=False, use_sequence_length=False) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e63c554e47..b7aa7bbf6b 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1978,6 +1978,7 @@ py_library( ":tensor_array_ops", ":util", ":variable_scope", + "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index dece290f83..e6848edc12 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2297,6 +2297,7 @@ cuda_py_test( "//tensorflow/python:control_flow_ops", "//tensorflow/python:data_flow_grad", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", "//tensorflow/python:init_ops", "//tensorflow/python:nn_grad", @@ -2305,6 +2306,7 @@ cuda_py_test( "//tensorflow/python:sparse_grad", "//tensorflow/python:tensor_array_grad", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", ], shard_count = 10, tags = ["no_windows"], diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py index a644e6a44f..d8f4b439e3 100644 --- a/tensorflow/python/kernel_tests/rnn_test.py +++ b/tensorflow/python/kernel_tests/rnn_test.py @@ -26,9 +26,12 @@ import numpy as np from tensorflow.contrib import rnn as contrib_rnn from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops as ops_lib from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl @@ -82,9 +85,13 @@ class RNNTest(test.TestCase): self._seed = 23489 np.random.seed(self._seed) + @test_util.run_in_graph_and_eager_modes() def testInvalidSequenceLengthShape(self): cell = Plus1RNNCell() - inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))] + if context.in_graph_mode(): + inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))] + else: + inputs = [constant_op.constant(np.ones((3, 4)))] with self.assertRaisesRegexp(ValueError, "must be a vector"): rnn.dynamic_rnn( cell, @@ -92,45 +99,77 @@ class RNNTest(test.TestCase): dtype=dtypes.float32, sequence_length=[[4]]) + @test_util.run_in_graph_and_eager_modes() def testBatchSizeFromInput(self): cell = Plus1RNNCell() + in_graph_mode = context.in_graph_mode() # With static batch size - inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) + if in_graph_mode: + inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) + initial_state = array_ops.placeholder(dtypes.float32, shape=(3, 5)) + else: + inputs = np.zeros((3, 4, 5), dtype=np.float32) + initial_state = np.zeros((3, 5), dtype=np.float32) + # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) - self.assertEqual(3, outputs.shape[0].value) - self.assertEqual(3, state.shape[0].value) + if in_graph_mode: + self.assertEqual(3, outputs.shape[0].value) + self.assertEqual(3, state.shape[0].value) + else: + self.assertEqual(3, outputs.shape[0]) + self.assertEqual(3, state.shape[0]) + # - With initial_state outputs, state = rnn.dynamic_rnn( - cell, - inputs, - initial_state=array_ops.placeholder(dtypes.float32, shape=(3, 5))) - self.assertEqual(3, outputs.shape[0].value) - self.assertEqual(3, state.shape[0].value) + cell, inputs, initial_state=initial_state) + if in_graph_mode: + self.assertEqual(3, outputs.shape[0].value) + self.assertEqual(3, state.shape[0].value) + else: + self.assertEqual(3, outputs.shape[0]) + self.assertEqual(3, state.shape[0]) + # Without static batch size - inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) - # - Without initial_state - outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) - self.assertEqual(None, outputs.shape[0].value) - self.assertEqual(None, state.shape[0].value) - # - With initial_state - outputs, state = rnn.dynamic_rnn( - cell, - inputs, - initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) - self.assertEqual(None, outputs.shape[0].value) - self.assertEqual(None, state.shape[0].value) + # Tensor shapes are fully determined in Eager mode, so only run this + # test in graph mode. + if in_graph_mode: + inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) + # - Without initial_state + outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) + self.assertEqual(None, outputs.shape[0].value) + self.assertEqual(None, state.shape[0].value) + # - With initial_state + outputs, state = rnn.dynamic_rnn( + cell, + inputs, + initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) + self.assertEqual(None, outputs.shape[0].value) + self.assertEqual(None, state.shape[0].value) + @test_util.run_in_graph_and_eager_modes() def testScalarStateIsAccepted(self): cell = ScalarStateRNNCell() - inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1)) + in_graph_mode = context.in_graph_mode() + + if in_graph_mode: + inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1)) + else: + inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32) + with self.test_session() as sess: outputs, state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32, sequence_length=[4]) - outputs, state = sess.run( - [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]}) - self.assertAllEqual(outputs, [[[1], [2], [3], [4]]]) - self.assertEqual(state, 4) + if in_graph_mode: + outputs, state = sess.run( + [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]}) + + if in_graph_mode: + self.assertAllEqual(outputs, np.array([[[1], [2], [3], [4]]])) + self.assertEqual(state, 4) + else: + self.assertAllEqual(outputs.numpy(), np.array([[[1], [2], [3], [4]]])) + self.assertEqual(state.numpy(), 4) ######### Benchmarking RNN code diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index b174956e60..21c7ed361d 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -27,6 +27,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -576,8 +577,9 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. with vs.variable_scope(scope or "rnn") as varscope: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) + if context.in_graph_mode(): + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) batch_size = _best_effort_input_batch_size(flat_input) if initial_state is not None: @@ -595,7 +597,7 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, ["Expected shape for Tensor %s is " % x.name, packed_shape, " but saw shape: ", x_shape]) - if sequence_length is not None: + if context.in_graph_mode() and sequence_length is not None: # Perform some shape validation with ops.control_dependencies( [_assert_has_shape(sequence_length, [batch_size])]): @@ -718,14 +720,19 @@ def _dynamic_rnn_loop(cell, size=time_steps, tensor_array_name=base_name + name) - output_ta = tuple(_create_ta("output_%d" % i, - _infer_state_dtype(dtype, state)) - for i in range(len(flat_output_size))) - input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype) - for i in range(len(flat_input))) - - input_ta = tuple(ta.unstack(input_) - for ta, input_ in zip(input_ta, flat_input)) + in_graph_mode = context.in_graph_mode() + if in_graph_mode: + output_ta = tuple(_create_ta("output_%d" % i, + _infer_state_dtype(dtype, state)) + for i in range(len(flat_output_size))) + input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype) + for i in range(len(flat_input))) + input_ta = tuple(ta.unstack(input_) + for ta, input_ in zip(input_ta, flat_input)) + else: + output_ta = tuple([0 for _ in range(time_steps.numpy())] + for i in range(len(flat_output_size))) + input_ta = flat_input def _time_step(time, output_ta_t, state): """Take a time step of the dynamic RNN. @@ -739,10 +746,13 @@ def _dynamic_rnn_loop(cell, The tuple (time + 1, output_ta_t with updated flow, new_state). """ - input_t = tuple(ta.read(time) for ta in input_ta) - # Restore some shape information - for input_, shape in zip(input_t, inputs_got_shape): - input_.set_shape(shape[1:]) + if in_graph_mode: + input_t = tuple(ta.read(time) for ta in input_ta) + # Restore some shape information + for input_, shape in zip(input_t, inputs_got_shape): + input_.set_shape(shape[1:]) + else: + input_t = tuple(ta[time.numpy()] for ta in input_ta) input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t) call_cell = lambda: cell(input_t, state) @@ -764,8 +774,12 @@ def _dynamic_rnn_loop(cell, # Pack state if using state tuples output = nest.flatten(output) - output_ta_t = tuple( - ta.write(time, out) for ta, out in zip(output_ta_t, output)) + if in_graph_mode: + output_ta_t = tuple( + ta.write(time, out) for ta, out in zip(output_ta_t, output)) + else: + for ta, out in zip(output_ta_t, output): + ta[time.numpy()] = out return (time + 1, output_ta_t, new_state) @@ -777,16 +791,20 @@ def _dynamic_rnn_loop(cell, swap_memory=swap_memory) # Unpack final output if not using output tuples. - final_outputs = tuple(ta.stack() for ta in output_final_ta) - - # Restore some shape information - for output, output_size in zip(final_outputs, flat_output_size): - shape = _concat( - [const_time_steps, const_batch_size], output_size, static=True) - output.set_shape(shape) + if in_graph_mode: + final_outputs = tuple(ta.stack() for ta in output_final_ta) + # Restore some shape information + for output, output_size in zip(final_outputs, flat_output_size): + shape = _concat( + [const_time_steps, const_batch_size], output_size, static=True) + output.set_shape(shape) + else: + final_outputs = output_final_ta final_outputs = nest.pack_sequence_as( structure=cell.output_size, flat_sequence=final_outputs) + if not in_graph_mode: + final_outputs = array_ops.stack(final_outputs, axis=0) return (final_outputs, final_state) @@ -967,8 +985,9 @@ def raw_rnn(cell, loop_fn, # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. with vs.variable_scope(scope or "rnn") as varscope: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) + if context.in_graph_mode(): + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) time = constant_op.constant(0, dtype=dtypes.int32) (elements_finished, next_input, initial_state, emit_structure, @@ -1166,8 +1185,9 @@ def static_rnn(cell, # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. with vs.variable_scope(scope or "rnn") as varscope: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) + if context.in_graph_mode(): + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) # Obtain the first sequence of the input first_input = inputs -- GitLab From 0f5683d629c6607d1baeaa44ecd264321ae05abc Mon Sep 17 00:00:00 2001 From: Jianwei Xie Date: Fri, 20 Oct 2017 10:45:51 -0700 Subject: [PATCH 202/573] Migrate the iris example to use TF core API. PiperOrigin-RevId: 172902682 --- tensorflow/examples/learn/iris.py | 101 ++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/tensorflow/examples/learn/iris.py b/tensorflow/examples/learn/iris.py index 33e8d45801..0a50b3ba87 100644 --- a/tensorflow/examples/learn/iris.py +++ b/tensorflow/examples/learn/iris.py @@ -17,47 +17,94 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np -from sklearn import datasets -from sklearn import metrics -from sklearn import model_selection +import os +import urllib import tensorflow as tf +# Data sets +IRIS_TRAINING = 'iris_training.csv' +IRIS_TRAINING_URL = 'http://download.tensorflow.org/data/iris_training.csv' -X_FEATURE = 'x' # Name of the input feature. +IRIS_TEST = 'iris_test.csv' +IRIS_TEST_URL = 'http://download.tensorflow.org/data/iris_test.csv' + +FEATURE_KEYS = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'] + + +def maybe_download_iris_data(file_name, download_url): + """Downloads the file and returns the number of data.""" + if not os.path.exists(file_name): + raw = urllib.urlopen(download_url).read() + with open(file_name, 'w') as f: + f.write(raw) + + # The first line is a comma-separated string. The first one is the number of + # total data in the file. + with open(file_name, 'r') as f: + first_line = f.readline() + num_elements = first_line.split(',')[0] + return int(num_elements) + + +def input_fn(file_name, num_data, batch_size, is_training): + """Creates an input_fn required by Estimator train/evaluate.""" + # If the data sets aren't stored locally, download them. + + def _parse_csv(rows_string_tensor): + """Takes the string input tensor and returns tuple of (features, labels).""" + # Last dim is the label. + num_features = len(FEATURE_KEYS) + num_columns = num_features + 1 + columns = tf.decode_csv(rows_string_tensor, + record_defaults=[[]] * num_columns) + features = dict(zip(FEATURE_KEYS, columns[:num_features])) + labels = tf.cast(columns[num_features], tf.int32) + return features, labels + + def _input_fn(): + """The input_fn.""" + dataset = tf.data.TextLineDataset([file_name]) + # Skip the first line (which does not have data). + dataset = dataset.skip(1) + dataset = dataset.map(_parse_csv) + + if is_training: + # For this small dataset, which can fit into memory, to achieve true + # randomness, the shuffle buffer size is set as the total number of + # elements in the dataset. + dataset = dataset.shuffle(num_data) + dataset = dataset.repeat() + + dataset = dataset.batch(batch_size) + iterator = dataset.make_one_shot_iterator() + features, labels = iterator.get_next() + return features, labels + + return _input_fn def main(unused_argv): - # Load dataset. - iris = datasets.load_iris() - x_train, x_test, y_train, y_test = model_selection.train_test_split( - iris.data, iris.target, test_size=0.2, random_state=42) + tf.logging.set_verbosity(tf.logging.INFO) + + num_training_data = maybe_download_iris_data( + IRIS_TRAINING, IRIS_TRAINING_URL) + num_test_data = maybe_download_iris_data(IRIS_TEST, IRIS_TEST_URL) # Build 3 layer DNN with 10, 20, 10 units respectively. feature_columns = [ - tf.feature_column.numeric_column( - X_FEATURE, shape=np.array(x_train).shape[1:])] + tf.feature_column.numeric_column(key, shape=1) for key in FEATURE_KEYS] classifier = tf.estimator.DNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3) # Train. - train_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True) - classifier.train(input_fn=train_input_fn, steps=200) - - # Predict. - test_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) - predictions = classifier.predict(input_fn=test_input_fn) - y_predicted = np.array(list(p['class_ids'] for p in predictions)) - y_predicted = y_predicted.reshape(np.array(y_test).shape) - - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - - # Score with tensorflow. + train_input_fn = input_fn(IRIS_TRAINING, num_training_data, batch_size=32, + is_training=True) + classifier.train(input_fn=train_input_fn, steps=400) + + # Eval. + test_input_fn = input_fn(IRIS_TEST, num_test_data, batch_size=32, + is_training=False) scores = classifier.evaluate(input_fn=test_input_fn) print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) -- GitLab From ff0530067435fea5c51605c2e7dfd55f6fe8dfe1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 11:06:03 -0700 Subject: [PATCH 203/573] Avoid silent variable sharing with ResourceVariable class. PiperOrigin-RevId: 172905986 --- tensorflow/contrib/eager/python/BUILD | 2 +- tensorflow/contrib/eager/python/saver_test.py | 13 +++----- tensorflow/python/eager/backprop_test.py | 5 +-- tensorflow/python/eager/function_test.py | 4 +-- .../resource_variable_ops_test.py | 32 ++++++++++++++----- .../python/ops/resource_variable_ops.py | 16 ++++++++++ tensorflow/python/training/saver_test.py | 3 +- 7 files changed, 53 insertions(+), 22 deletions(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 702136e3e4..ace17424fe 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -86,7 +86,7 @@ cuda_py_test( "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python/eager:graph_callable", - "//tensorflow/python:platform_test", + "//tensorflow/python/eager:test", "//tensorflow/python:variables", ], ) diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py index 29af2b531f..c89554e6dd 100644 --- a/tensorflow/contrib/eager/python/saver_test.py +++ b/tensorflow/contrib/eager/python/saver_test.py @@ -22,6 +22,7 @@ import os from tensorflow.contrib.eager.python import saver as _saver from tensorflow.python.eager import context from tensorflow.python.eager import graph_callable +from tensorflow.python.eager import test from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops @@ -29,7 +30,6 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import test class SaverTest(test.TestCase): @@ -38,7 +38,7 @@ class SaverTest(test.TestCase): return '/device:GPU:0' if context.num_gpus() else '/device:CPU:0' def testBasics(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') def model(): return array_ops.constant(2.0) * v1 @@ -55,7 +55,7 @@ class SaverTest(test.TestCase): self.assertEqual(v1.read_value().numpy(), 1.0) def testRestoreOnCreate(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): def model(init_val): v1 = resource_variable_ops.ResourceVariable(init_val, name='v1') return array_ops.constant(1.0) * v1, v1 @@ -71,12 +71,9 @@ class SaverTest(test.TestCase): # Value is from checkpoint, but not from argument. ret, _ = model(2.0) self.assertEqual(ret.numpy(), 1.0) - # Create it a second time won't re-assign the checkpoint value. - v1_2 = resource_variable_ops.ResourceVariable(3.0, name='v1') - self.assertEqual(v1_2.read_value().numpy(), 3.0) def testRestoreNotFound(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): def model(v): return array_ops.constant(1.0) * v @@ -92,7 +89,7 @@ class SaverTest(test.TestCase): _ = model(resource_variable_ops.ResourceVariable(1.0, name='v2')) def testSaveRestoreGraphCallable(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): @graph_callable.graph_callable( [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)]) def model(x): diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 7da8eb0c9b..9ba5913c65 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -292,7 +292,7 @@ class BackpropTest(test.TestCase): self.assertEqual(grad.numpy(), 6.0) def testGradientTapeVariable(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name='v') with backprop.GradientTape() as g: y = v * v grad = g.gradient(y, [v])[0] @@ -457,7 +457,8 @@ class BackpropTest(test.TestCase): add_n.append(1) context.context().add_post_execution_callback(callback) - v = resource_variable_ops.ResourceVariable(constant_op.constant(2.0)) + v = resource_variable_ops.ResourceVariable(constant_op.constant(2.0), + name='v') def fn(): outputs = [] for _ in range(20): diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index a4c351e8c9..33bedb59f3 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -57,7 +57,7 @@ class FunctionTest(test.TestCase): self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) def testGraphModeWithGradients(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name='v') @function.defun def step(): @@ -156,7 +156,7 @@ class FunctionTest(test.TestCase): g(constant_op.constant(1.0)) def testGradientTensorConversionWithDefun(self): - three = resource_variable_ops.ResourceVariable(3.0) + three = resource_variable_ops.ResourceVariable(3.0, name='v') @function.defun def f(x): diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index cf4b61674f..10f9a72c7b 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -181,7 +181,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testInitFnDtype(self): v = resource_variable_ops.ResourceVariable( - initial_value=lambda: 1, dtype=dtypes.float32) + initial_value=lambda: 1, dtype=dtypes.float32, name="var0") self.assertEqual(dtypes.float32, v.value().dtype) @test_util.run_in_graph_and_eager_modes() @@ -192,26 +192,27 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testInitializeAllVariables(self): - v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.float32) + v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.float32, + name="var0") self.evaluate(variables.global_variables_initializer()) self.assertEqual(1.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testOperatorOverload(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.assertEqual(2.0, self.evaluate(v + v)) @test_util.run_in_graph_and_eager_modes() def testAssignMethod(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.evaluate(v.assign(2.0)) self.assertEqual(2.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testLoad(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) v.load(2.0) self.assertEqual(2.0, self.evaluate(v.value())) @@ -237,21 +238,21 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testAssignAddMethod(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.evaluate(v.assign_add(1.0)) self.assertEqual(2.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testAssignSubMethod(self): - v = resource_variable_ops.ResourceVariable(3.0) + v = resource_variable_ops.ResourceVariable(3.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.evaluate(v.assign_sub(1.0)) self.assertEqual(2.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testDestroyResource(self): - v = resource_variable_ops.ResourceVariable(3.0) + v = resource_variable_ops.ResourceVariable(3.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.assertEqual(3.0, self.evaluate(v.value())) self.evaluate(resource_variable_ops.destroy_resource_op(v.handle)) @@ -443,6 +444,21 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): resource_variable_ops.destroy_resource_op(var._handle, ignore_lookup_error=False) + def testSharingViaResourceVariableObject(self): + with context.eager_mode(): + _ = resource_variable_ops.ResourceVariable(1.0, name="var0") + with self.assertRaisesRegexp(ValueError, + "'var0' already created"): + _ = resource_variable_ops.ResourceVariable(2.0, name="var0") + with ops.Graph().as_default(): + _ = resource_variable_ops.ResourceVariable(2.0, name="var0") + + def testVariableNameMissing(self): + with context.eager_mode(): + with self.assertRaisesRegexp(ValueError, + "Variables need to have explicit names"): + _ = resource_variable_ops.ResourceVariable(1.0) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index aa45752a9d..c94ddb0627 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -49,6 +49,16 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): container=container) if graph_mode: return handle + + # We do not want two distinct ResourceVariable objects for the same + # underlying resource in the runtime. + # When in eager mode, explicitly ensure so here. When in graph mode, it's + # ensured by always generating different variable names. + exists = gen_resource_variable_ops.var_is_initialized_op(handle) + if exists: + raise ValueError("variable object with name '%s' already created. Use " + "get_variable() if reuse is desired." % + shared_name) with context.graph_mode(), ops.Graph().as_default(): h = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype, shared_name=shared_name, @@ -273,6 +283,12 @@ class ResourceVariable(variables.Variable): # Save the graph's container prefix for error checking. Reading the value of # the ResourceVariable from another Graph in Eager mode is an error. self._container_prefix = ops.get_default_graph()._container_prefix # pylint: disable=protected-access + if not self._in_graph_mode and not name: + # TODO(ashankar,josh11b): make this unnecessary using the same + # logic as in layer + raise ValueError("Variables need to have explicit names when eager " + "execution is enabled") + with ops.control_dependencies(None): with ops.name_scope(name, "Variable", [] if init_from_fn else [initial_value]) as name: diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index aeb8eaffe8..4abff1d106 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -233,7 +233,8 @@ class SaverTest(test.TestCase): def testResourceSaveRestoreCachingDevice(self): save_path = os.path.join(self.get_temp_dir(), "resource_cache") with self.test_session(graph=ops_lib.Graph()) as sess: - v = resource_variable_ops.ResourceVariable([1], caching_device="/cpu:0") + v = resource_variable_ops.ResourceVariable([1], caching_device="/cpu:0", + name="v") if context.in_graph_mode(): self.evaluate(variables.global_variables_initializer()) else: -- GitLab From 017a5021a7fdc713357fceecf31068ae5090afaf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 11:13:03 -0700 Subject: [PATCH 204/573] [XLA:CPU] Do not assign parallel tasks to instructions which forward pointers (GetTupleElement and Bitcast), because the process of outlining the instruction into a parallel computation forces the pointed-to buffer to be materialized. PiperOrigin-RevId: 172907063 --- tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc index 5afb2e67ff..c2213c8f2e 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc +++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc @@ -136,6 +136,8 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount( instruction->opcode() == HloOpcode::kCall || instruction->opcode() == HloOpcode::kCustomCall || instruction->opcode() == HloOpcode::kSelectAndScatter || + instruction->opcode() == HloOpcode::kGetTupleElement || + instruction->opcode() == HloOpcode::kBitcast || (instruction->opcode() == HloOpcode::kConvolution && PotentiallyImplementedAsEigenConvolution(*instruction)) || PotentiallyImplementedAsEigenDot(*instruction) || -- GitLab From 86908c30c4c0adf92fa14ed6f1d92616177c1b89 Mon Sep 17 00:00:00 2001 From: Jianwei Xie Date: Fri, 20 Oct 2017 11:13:45 -0700 Subject: [PATCH 205/573] Step 1: Large refactoring toward wrapping input_fn and TPU infeed into tf.while_loop PiperOrigin-RevId: 172907182 --- .../contrib/tpu/python/tpu/tpu_estimator.py | 1313 +++++++++-------- 1 file changed, 664 insertions(+), 649 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 04e0719a1b..805de16468 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import print_function import collections +from contextlib import contextmanager import copy import threading import six @@ -38,6 +39,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.estimator import estimator as estimator_lib from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator import util +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -57,12 +59,15 @@ from tensorflow.python.training import training_util _INITIAL_LOSS = 1e7 _ZERO_LOSS = 0. -_DEFAULT_NAME_SCOPE = 'tpu_estimator' +_TPU_ESTIMATOR = 'tpu_estimator' _ITERATIONS_PER_LOOP_VAR = 'iterations_per_loop' _BATCH_SIZE_KEY = 'batch_size' _CROSS_REPLICA_SUM_OP = 'CrossReplicaSum' _RESERVED_PARAMS_KEYS = [_BATCH_SIZE_KEY] +# TODO(b/65703635): Flip the value and remove all dead code. +_WRAP_INPUT_FN_INTO_WHILE_LOOP = False + def _create_global_step(graph): graph = graph or ops.get_default_graph() @@ -81,17 +86,25 @@ def _create_global_step(graph): ops.GraphKeys.GLOBAL_STEP]) -def _create_iterations_per_loop(): - with variable_scope.variable_scope(_DEFAULT_NAME_SCOPE, - reuse=variable_scope.AUTO_REUSE): - return variable_scope.get_variable( - _ITERATIONS_PER_LOOP_VAR, - initializer=init_ops.zeros_initializer(), - shape=[], - dtype=dtypes.int32, - trainable=False, - collections=[], - use_resource=True) +def _create_or_get_iterations_per_loop(): + graph = ops.get_default_graph() + iter_vars = graph.get_collection(_TPU_ESTIMATOR) + if len(iter_vars) == 1: + return iter_vars[0] + elif len(iter_vars) > 1: + raise RuntimeError('Multiple iterations_per_loop_var in collection.') + + with ops.colocate_with(training_util.get_global_step()): + with variable_scope.variable_scope(_TPU_ESTIMATOR, + reuse=variable_scope.AUTO_REUSE): + return variable_scope.get_variable( + _ITERATIONS_PER_LOOP_VAR, + initializer=init_ops.zeros_initializer(), + shape=[], + dtype=dtypes.int32, + trainable=False, + collections=[_TPU_ESTIMATOR], + use_resource=True) def _sync_variables_ops(): @@ -127,64 +140,209 @@ _DEFAULT_COORDINATOR_JOB_NAME = 'coordinator' _LOCAL_MASTERS = ('', 'local') -def _tpu_job(run_config, mode): - """Returns the job name to use to place TPU computations on. - - Args: - run_config: The tpu_config.RunConfig used for this custom estimator. - mode: A model_fn_lib.ModeKeys value. +class _TPUContext(object): + """A context holds immutable states of TPU computation. - Returns: - A string containing the job name, or None if no job should be specified. + This immutable object holds TPUEstimator config, train/eval batch size, and + `TPUEstimator.use_tpu`, which is expected to be passed around. It also + provides utility functions, basded on the current state, to determine other + information commonly required by TPU computation, such as TPU device names, + TPU hosts, shard batch size, etc. - Raises: - ValueError: If the user needs to specify a tpu_job_name, because we are - unable to infer the job name automatically, or if the user-specified job - names are inappropriate. + N.B. As `mode` is not immutable state in Estimator, but essential to + distinguish between TPU training and evaluation, a common usage for + _TPUContext with `mode` is as follows: + ``` + with _ctx.with_mode(mode) as ctx: + if ctx.is_running_on_cpu(): + ... + ``` """ - # If the user specifies the tpu_job_name, use that. - if run_config.tpu_config.tpu_job_name: - return run_config.tpu_config.tpu_job_name - - # The tpu job is determined by the run_config. Right now, this method is - # required as tpu_config is not part of the RunConfig. - master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL - else run_config.master) - if master in _LOCAL_MASTERS: - return None - - if (not run_config.session_config or - not run_config.session_config.cluster_def.job): - return _DEFAULT_JOB_NAME - cluster_def = run_config.session_config.cluster_def - job_names = set([job.name for job in cluster_def.job]) - if _DEFAULT_JOB_NAME in job_names: - # b/37868888 tracks allowing ClusterSpec propagation to reuse job names. - raise ValueError('Currently, tpu_worker is not an allowed job name.') - if len(job_names) == 1: - return cluster_def.job[0].name - if len(job_names) == 2: - if _DEFAULT_COORDINATOR_JOB_NAME in job_names: - job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME) - return job_names.pop() - # TODO(b/67716447): Include more sophisticated heuristics. - raise ValueError( - 'Could not infer TPU job name. Please specify a tpu_job_name as part of ' - 'your TPUConfig.') - - -def _is_running_on_cpu(use_tpu, mode, eval_batch_size): - """Determines whether the input_fn and model_fn should be invoked on CPU.""" - return ((not use_tpu) or mode == model_fn_lib.ModeKeys.PREDICT or - (mode == model_fn_lib.ModeKeys.EVAL and eval_batch_size is None)) - - -def _per_shard_batch_size(global_batch_size, run_config, use_tpu): - """Returns the batch size for each shard.""" - if use_tpu: - return global_batch_size // run_config.tpu_config.num_shards - else: - return global_batch_size + + def __init__(self, config, train_batch_size, eval_batch_size, use_tpu): + self._config = config + self._train_batch_size = train_batch_size + self._eval_batch_size = eval_batch_size + self._use_tpu = use_tpu + self._num_shards_or_none = self._config.tpu_config.num_shards + self._mode = None + + def _assert_mode(self): + if self._mode is None: + raise RuntimeError( + '`mode` needs to be set via contextmanager `with_mode`.') + return self._mode + + @property + def num_of_cores_per_host(self): + num_cores = self.num_cores + return min(num_cores, 8) + + @contextmanager + def with_mode(self, mode): + new_ctx = copy.copy(self) # Shallow copy is enough. + new_ctx._mode = mode # pylint: disable=protected-access + yield new_ctx + + @property + def mode(self): + return self._assert_mode() + + @property + def num_cores(self): + # TODO(xiejw): Adds lazy num_shards initialization. + return self._num_shards_or_none + + @property + def num_hosts(self): + return self.num_cores // self.num_of_cores_per_host + + @property + def config(self): + return self._config + + def is_input_sharded_per_core(self): + """Return true if input_fn is invoked per-core (other than per-host).""" + self._assert_mode() + return (self._mode == model_fn_lib.ModeKeys.TRAIN and + not self._config.tpu_config.per_host_input_for_training) + + def is_running_on_cpu(self): + """Determines whether the input_fn and model_fn should be invoked on CPU.""" + mode = self._assert_mode() + return ((not self._use_tpu) or mode == model_fn_lib.ModeKeys.PREDICT or + (mode == model_fn_lib.ModeKeys.EVAL and + self._eval_batch_size is None)) + + @property + def batch_size_for_input_fn(self): + """Returns the shard batch size for `input_fn`.""" + mode = self._assert_mode() + # Special case for eval. + if mode == model_fn_lib.ModeKeys.EVAL and self._eval_batch_size is None: + return None + if self.is_running_on_cpu(): + if mode == model_fn_lib.ModeKeys.TRAIN: + return self._train_batch_size + if mode == model_fn_lib.ModeKeys.EVAL: + return self._eval_batch_size + return None + + global_batch_size = (self._train_batch_size if + mode == model_fn_lib.ModeKeys.TRAIN + else self._eval_batch_size) + # On TPU + return (global_batch_size // self.num_cores + if self.is_input_sharded_per_core() else global_batch_size) + + @property + def batch_size_for_model_fn(self): + """Returns the shard batch size for `model_fn`.""" + mode = self._assert_mode() + # Special case for eval. + if mode == model_fn_lib.ModeKeys.EVAL and self._eval_batch_size is None: + return None + if self.is_running_on_cpu(): + if mode == model_fn_lib.ModeKeys.TRAIN: + return self._train_batch_size + if mode == model_fn_lib.ModeKeys.EVAL: + return self._eval_batch_size + return None + + # On TPU. always sharded per core. + if mode == model_fn_lib.ModeKeys.TRAIN: + return self._train_batch_size // self.num_cores + else: + return self._eval_batch_size // self.num_cores + + @property + def master_job(self): + """Returns the job name to use to place TPU computations on. + + Returns: + A string containing the job name, or None if no job should be specified. + + Raises: + ValueError: If the user needs to specify a tpu_job_name, because we are + unable to infer the job name automatically, or if the user-specified job + names are inappropriate. + """ + run_config = self._config + # If the user specifies the tpu_job_name, use that. + if run_config.tpu_config.tpu_job_name: + return run_config.tpu_config.tpu_job_name + + # The tpu job is determined by the run_config. Right now, this method is + # required as tpu_config is not part of the RunConfig. + mode = self._assert_mode() + master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL + else run_config.master) + if master in _LOCAL_MASTERS: + return None + + if (not run_config.session_config or + not run_config.session_config.cluster_def.job): + return _DEFAULT_JOB_NAME + cluster_def = run_config.session_config.cluster_def + job_names = set([job.name for job in cluster_def.job]) + if _DEFAULT_JOB_NAME in job_names: + # b/37868888 tracks allowing ClusterSpec propagation to reuse job names. + raise ValueError('Currently, tpu_worker is not an allowed job name.') + if len(job_names) == 1: + return cluster_def.job[0].name + if len(job_names) == 2: + if _DEFAULT_COORDINATOR_JOB_NAME in job_names: + job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME) + return job_names.pop() + # TODO(b/67716447): Include more sophisticated heuristics. + raise ValueError( + 'Could not infer TPU job name. Please specify a tpu_job_name as part ' + 'of your TPUConfig.') + + @property + def tpu_host_placement_function(self): + """Returns the TPU host place function.""" + master = self.master_job + def _placement_function(_sentinal=None, core_id=None, host_id=None): # pylint: disable=invalid-name + assert _sentinal is None + if core_id is not None and host_id is not None: + raise RuntimeError( + 'core_id and host_id can have only one non-None value.') + + if master is None: + return '/replica:0/task:0/device:CPU:0' + else: + # This assumes that if using more than 8 shards, + # the job configuration varies 'task'. + if core_id is not None: + host_id = core_id / 8 + return '/job:%s/task:%d/device:CPU:0' % (master, host_id) + return _placement_function + + @property + def tpu_device_placement_function(self): + master = self.master_job + job_device = '' if master is None else ('/job:%s' % master) + def _placement_function(i): + return '%s/task:%d/device:TPU:%d' % (job_device, i / 8, i % 8) + return _placement_function + + @property + def tpu_ordinal_function(self): + """Returns the TPU ordinal fn.""" + def _tpu_ordinal_function(index): + """Return the TPU ordinal associated with a shard. + + Required because the enqueue ops are placed on CPU. + + Args: + index: the shard index + + Returns: + The ordinal of the TPU device the shard's infeed should be placed on. + """ + return index % 8 + return _tpu_ordinal_function class _SIGNAL(object): @@ -319,11 +477,16 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController): logging.info('Stop Infeed input thread.') return - iterations = signal - for i in range(iterations): - logging.debug('Infeed enqueue for iteration (%d, %d)', count, i) + if _WRAP_INPUT_FN_INTO_WHILE_LOOP: + # Enqueue batches for next loop. session.run(enqueue_ops) - count += 1 + else: + iterations = signal + for i in range(iterations): + logging.debug('Infeed enqueue for iteration (%d, %d)', count, i) + session.run(enqueue_ops) + count += 1 + except Exception: # pylint: disable=broad-except logging.error( 'Failed running infeed, closing session.\n' @@ -346,17 +509,16 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): dequeue. """ - def __init__(self, run_config, mode, enqueue_fn, dequeue_ops=None): - self._tpu_job = _tpu_job(run_config, mode) - self._enqueue_fn = enqueue_fn + def __init__(self, ctx, enqueue_ops, dequeue_ops=None): + self._master_job = ctx.master_job + self._enqueue_ops = enqueue_ops self._dequeue_ops = dequeue_ops def begin(self): - self._enqueue_ops = self._enqueue_fn() - self._iterations_per_loop_var = _create_iterations_per_loop() - logging.info('TPU job name %s', self._tpu_job) - self._init_op = [tpu.initialize_system(job=self._tpu_job)] - self._finalize_op = [tpu.shutdown_system(job=self._tpu_job)] + logging.info('TPU job name %s', self._master_job) + self._iterations_per_loop_var = _create_or_get_iterations_per_loop() + self._init_op = [tpu.initialize_system(job=self._master_job)] + self._finalize_op = [tpu.shutdown_system(job=self._master_job)] def after_create_session(self, session, coord): logging.info('Init TPU system') @@ -378,6 +540,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): iterations = run_context.session.run(self._iterations_per_loop_var) self._infeed_thd_controller.send_next_batch_signal(iterations) if self._dequeue_ops is not None: + # TODO(xiejw): Refactor the outfeed dequeue into tf.while_loop. logging.info('Dequeue next batch of data from outfeed.') self._outfeed_thd_controller.send_next_batch_signal(iterations) @@ -439,7 +602,7 @@ class _TPUStopAtStepHook(session_run_hook.SessionRunHook): if self._global_step_tensor is None: raise RuntimeError('Global step should be created.') - self._iterations_per_loop_var = _create_iterations_per_loop() + self._iterations_per_loop_var = _create_or_get_iterations_per_loop() def after_create_session(self, session, coord): global_step = session.run(self._global_step_tensor) @@ -474,360 +637,288 @@ class _SetEvalIterationsHook(session_run_hook.SessionRunHook): self._num_steps = num_steps def begin(self): - self._iterations_per_loop_var = _create_iterations_per_loop() + self._iterations_per_loop_var = _create_or_get_iterations_per_loop() def after_create_session(self, session, coord): self._iterations_per_loop_var.load(self._num_steps, session=session) -class _PerShardOutput(object): - """Wraps input_fn's outputs into per-shard outputs. - - Used so that the model_fn can distinguish between sharded input and unsharded - inputs (e.g., for export_savedmodel()). - """ - - def __init__(self, output): - self.output = output - - def as_list(self): - return self.output - +def generate_per_core_enqueue_ops_fn_for_host( + ctx, input_fn, inputs_structure_recorder): + """Generates infeed enqueue ops for per-core input_fn on a single host.""" + infeed_queue_holder = {'instance': None} + + def enqueue_ops_fn(): + """A fn returns enqueue_ops.""" + num_cores_per_host = ctx.num_of_cores_per_host + per_host_sharded_inputs = [] + for core_ordinal in range(num_cores_per_host): + with ops.name_scope('ordinal_%d' % (core_ordinal)): + inputs = input_fn() + if isinstance(inputs, tuple): + features, labels = inputs + else: + features, labels = inputs, None -class _InputsHolder(object): - """A inputs holder holds the `features` and `labels' for TPU system. + inputs_structure_recorder.validate_and_record_structure( + features, labels) + flattened_inputs = ( + inputs_structure_recorder.flatten_features_and_labels( + features, labels)) + per_host_sharded_inputs.append(flattened_inputs) - Model inputs returned by the `input_fn` can have one of the following forms: + infeed_queue = tpu_feed.InfeedQueue( + number_of_tuple_elements=len(per_host_sharded_inputs[0])) + infeed_queue_holder['instance'] = infeed_queue + infeed_queue.set_configuration_from_sharded_input_tensors( + per_host_sharded_inputs) + + per_host_enqueue_ops = infeed_queue.generate_enqueue_ops( + per_host_sharded_inputs, + tpu_ordinal_function=ctx.tpu_ordinal_function) + return per_host_enqueue_ops + return enqueue_ops_fn, (lambda: infeed_queue_holder['instance']) + + +class _InputPipeline(object): + """`_InputPipeline` handles invoking `input_fn` and piping to infeed queue. + + `_InputPipeline` abstracts the per-core/per-host `input_fn` invocation from + call site. To be precise, based on the configuration in `_TPUContext`, it + invokes `input_fn` for all cores (usually multi-host TPU training) or for one + host (usually for single-host TPU evaluation), and sends all `features` and + `labels` returned by `input_fn` to TPU infeed. For per-core invocation, + `features` and `labels` are piped to infeed directly, one tuple for each + core. For per-host invocation, `features` and `labels` are split at host + (with respect to `batch_axis`) and piped to all cores accordingly. + + In addition, flatten/unflatten are handled by `_InputPipeline` also. Model + inputs returned by the `input_fn` can have one of the following forms: 1. features 2. (features, labels) Internally, form 1 is reformed to `(features, None)` as features and labels are passed separatedly to underlying methods. For TPU training, TPUEstimator - expects multiple `features` and `labels` tuples one for each shard. - - In addition, TPUEstimator allows various different structures for inputs - (namely `features` and `labels`). `features` can be `Tensor` or dict of - string name to `Tensor`, and `labels` could be `None`, `Tensor`, or dict of - string name to `Tensor`. TPU infeed/outfeed library expects flattened tensor - list. So, `features` and `labels` need to be flattened, before infeed enqueue, - and the structure of them needs to be recorded, in order to restore them after - infeed dequeue. - - `_InputsHolder` could hold the `features` and `labels` tuple for all shards - (usually multi-host TPU training) or for one host (usually for single-host TPU - evaluation), records the structure details (including presence, dict or single - tensor, dict names), validates the structure consistency cross all shards, and - encapsulates the flatten/unflatten logic. + may expect multiple `features` and `labels` tuples one for each core. + + TPUEstimator allows various different structures for inputs (namely `features` + and `labels`). `features` can be `Tensor` or dict of string name to `Tensor`, + and `labels` could be `None`, `Tensor`, or dict of string name to `Tensor`. + TPU infeed/outfeed library expects flattened tensor list. So, `features` and + `labels` need to be flattened, before infeed enqueue, and the structure of + them needs to be recorded, in order to restore them after infeed dequeue. """ - def __init__(self, features=None, labels=None, num_shards=None): - """Constructor. - - Args: - features: features for one host or a list of features one for each shard - (must be type `_PerShardOutput`). Once provided, the corresponding - `labels` should be set also and this `_InputsHolder` is frozen to - prevent from future modification. If `None`, it is expected to add - features and labels for each shard by calling `append_tuple` later. - labels: labels for one host or a list of labels one for each shard - (must be type `_PerShardOutput`). - num_shards: Number of shards in the TPU system. Must be provided unless it - can be deduced from `features`. - - Raises: - ValueError: If both `sharded_features` and `num_shards` are `None`. - """ - # Holds the features and labels for all shards. - self._feature_list = [] - self._label_list = [] - - # Holds the structure of inputs - self._feature_names = [] - self._label_names = [] - self._has_labels = False - - # Internal state. - self._initialized = False - self._frozen = False - self._sharded = False - - if features is None: - if num_shards is None: - raise ValueError( - '`features` and `num_shards` cannot be both None') - self._num_shards = num_shards - elif isinstance(features, _PerShardOutput): - self._from_sharded_inputs(features, labels, num_shards) - else: - if num_shards is None: - raise ValueError( - '`num_shards` cannot be None for unsharded features.') - self._from_unsharded_inputs(features, labels, num_shards) - - def _from_unsharded_inputs(self, features, labels, num_shards): - """Initializes the inputs with unsharded features and labels.""" - self._num_shards = num_shards - if labels is not None: - self._has_labels = True - self.append_tuple((features, labels)) - else: - self.append_tuple(features) - - self._sharded = False - self._frozen = True - - def _from_sharded_inputs(self, sharded_features, sharded_labels, num_shards): - """Initializes the inputs with sharded features and labels.""" - if not isinstance(sharded_features, _PerShardOutput): - raise ValueError('`sharded_features` must have type `_PerShardOutput`.') - features = sharded_features.as_list() - - if num_shards is not None and num_shards != len(features): - raise ValueError( - '`num_shards` should be same as the length of sharded_features.') + class InputsStructureRecorder(object): + """The recorder to record inputs structure.""" + + def __init__(self): + # Holds the structure of inputs + self._feature_names = [] + self._label_names = [] + self._has_labels = False + + # Internal state. + self._initialized = False + + def has_labels(self): + return self._has_labels + + def validate_and_record_structure(self, features, labels): + """Validates and records the structure of features` and `labels`.""" + def _extract_key_names(tensor_or_dict): + if tensor_or_dict is None: + return [] + return tensor_or_dict.keys() if isinstance(tensor_or_dict, dict) else [] + + # Extract structure. + has_labels = labels is not None + feature_names = _extract_key_names(features) + label_names = _extract_key_names(labels) + + if self._initialized: + # Verify the structure is same. The following should never happen. + assert feature_names == self._feature_names, 'feature keys mismatched' + assert label_names == self._label_names, 'label keys mismatched' + assert has_labels == self._has_labels, 'label presence mismatched' + else: + # Record structure. + self._initialized = True + self._feature_names = feature_names + self._label_names = label_names + self._has_labels = has_labels + + def flatten_features_and_labels(self, features, labels): + """Flattens the `features` and `labels` to a single tensor list.""" + flattened_inputs = [] + if self._feature_names: + # We need a fixed ordering for enqueueing and dequeueing. + flattened_inputs.extend([features[name] + for name in self._feature_names]) + else: + flattened_inputs.append(features) - self._num_shards = len(features) - if not self._num_shards: - raise ValueError('`sharded_features` should not be empty.') + if labels is not None: + if self._label_names: + # We need a fixed ordering for enqueueing and dequeueing. + flattened_inputs.extend([labels[name] for name in self._label_names]) + else: + flattened_inputs.append(labels) + return flattened_inputs + + def unflatten_features_and_labels(self, flattened_inputs): + """Restores the flattened inputs to original features and labels form. + + Args: + flattened_inputs: Flattened inputs for each shard. + + Returns: + A tuple of (`features`, `labels`), where `labels` could be None. + Each one, if present, should have identical structure (single tensor vs + dict) as the one returned by input_fn. + + Raises: + ValueError: If the number of expected tensors from `flattened_inputs` + mismatches the recorded structure. + """ + expected_num_features = (len(self._feature_names) if self._feature_names + else 1) + if self._has_labels: + expected_num_labels = (len(self._label_names) if self._label_names + else 1) + else: + expected_num_labels = 0 - if sharded_labels is not None: - if not isinstance(sharded_labels, _PerShardOutput): - raise ValueError('sharded_labels` must have type `_PerShardOutput`.') + expected_num_tensors = expected_num_features + expected_num_labels - self._has_labels = True - labels = sharded_labels.as_list() - if self._num_shards != len(labels): + if expected_num_tensors != len(flattened_inputs): raise ValueError( - 'Length of `sharded_features` and `sharded_labels` mismatch.') - - if self._has_labels: - for (f, l) in zip(features, labels): - self.append_tuple((f, l)) - else: - for f in features: - self.append_tuple(f) - - self._sharded = True - self._frozen = True - - def _extract_key_names(self, tensor_or_dict): - if tensor_or_dict is None: - return [] - - return tensor_or_dict.keys() if isinstance(tensor_or_dict, dict) else [] - - def _validate(self, features, labels): - has_labels = labels is not None - feature_names = self._extract_key_names(features) - label_names = self._extract_key_names(labels) - - if self._initialized: - self._sharded = True - # The following should never happen. - assert feature_names == self._feature_names, 'feature keys mismatched' - assert label_names == self._label_names, 'label keys mismatched' - assert has_labels == self._has_labels, 'label presence mismatched' - else: - self._initialized = True - self._feature_names = feature_names - self._label_names = label_names - self._has_labels = has_labels - - @property - def sharded(self): - if not self._frozen: - raise RuntimeError('_InputsHolder has not been frozen yet.') - return self._sharded - - @property - def num_shards(self): - if not self._frozen: - raise RuntimeError('_InputsHolder has not been frozen yet.') - return self._num_shards - - def append_tuple(self, inputs): - """Appends `inputs` for one shard into holder. - - Args: - inputs: The return from `input_fn`, which could be features or tuple of - (features, labels). After the first `inputs` appended into - `_InputsHolder`, the structure of `features` and `labels is recorded. - Any future invocation should provide the `inputs` with same structure. - - Raises: - RuntimeError: If the internal data has been frozen already. - """ - if self._frozen: - raise RuntimeError('InputsHolder has frozen, which cannot be mutated.') - - # input_fn may return either features or (features, labels) - if isinstance(inputs, tuple): - features, labels = inputs - else: - features, labels = inputs, None - - self._validate(features, labels) - - self._feature_list.append(features) - if labels is not None: - self._label_list.append(labels) - - def as_features_and_labels_tuple(self): - """Returns features and labels as grouped tuple. - - This is intended to be used to pass features and labels for all shards from - input_fn to model_fn as the parent class `Estimator` does not have the - concept of shards. So, grouped tuple is required. - - Once called, the internal data is frozen and `append_tuple` cannot be - invoked anymore. - - Returns: - A tuple of features and labels. Both have type `_PerShardOutput`, holding - the inputs for all shards. `labels` could be `None`. - - Raises: - RuntimeError: If the internal data has not been initialized. - """ - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - - assert len(self._feature_list) == self._num_shards - if not self._label_list or all(l is None for l in self._label_list): - return _PerShardOutput(self._feature_list), None - - assert len(self._label_list) == self._num_shards - return (_PerShardOutput(self._feature_list), - _PerShardOutput(self._label_list)) - - def as_sharded_flattened_inputs(self): - """Flatten the features and label as tensor lists for all shards. - - Flattened tensor list contains all tensors in `features` (dict) and `labels` - (dict). Conceptually, it has the predicated structure like: - - ```python - flatten_list = [] - for name in features: - flatten_list.append(features[name]) - for name in labels: - flatten_list.append(labels[name]) - ``` - - This method handles the label is None case and single tensor case nicely. - - Once called, the internal data is frozen and `append_tuple` cannot be - invokded anymore. - - Returns: - A list of flattened inputs one for each shard. - - Raises: - RuntimeError: If the internal data has not been initialized. - ValueError: If the inputs are sharded. - """ - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - if not self._sharded: - raise ValueError('Inputs are not sharded.') - - sharded_inputs = [] - - for shard in range(self._num_shards): - flattened_inputs = self._as_flattened_inputs( - self._feature_list[shard], - self._label_list[shard] if self._has_labels else None) - sharded_inputs.append(flattened_inputs) - - return sharded_inputs - - def as_flattened_inputs(self): - """Flatten the features and label as a single tensor list for one host.""" - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - if self._sharded: - raise ValueError('Inputs are sharded.') - - return self._as_flattened_inputs( - self._feature_list[0], - self._label_list[0] if self._has_labels else None) - - def _as_flattened_inputs(self, features, labels): - """Flattens the `features` and `labels` to a single tensor list.""" - flattened_inputs = [] - if self._feature_names: - # We need a fixed ordering for enqueueing and dequeueing. - flattened_inputs.extend([features[name] for name in self._feature_names]) - else: - flattened_inputs.append(features) - - if labels is not None: - if self._label_names: - # We need a fixed ordering for enqueueing and dequeueing. - flattened_inputs.extend([labels[name] for name in self._label_names]) + 'The number of flattened tensors mismatches expected num. ' + 'Expected {}, got {}'.format(expected_num_tensors, + len(flattened_inputs))) + if self._feature_names: + unflattened_features = dict( + zip(self._feature_names, flattened_inputs[:expected_num_features])) + else: + # Single tensor case + unflattened_features = flattened_inputs[0] + + if expected_num_labels == 0: + unflattened_label = None + elif self._label_names: + unflattened_label = dict(zip(self._label_names, + flattened_inputs[expected_num_features:])) else: - flattened_inputs.append(labels) - return flattened_inputs + # Single tensor case. + unflattened_label = flattened_inputs[expected_num_features] - def unflatten_features_and_labels(self, flattened_inputs): - """Restores the flattened inputs to original features and labels form. + return unflattened_features, unflattened_label - Once called, the internal data is frozen and `append_tuple` cannot be - invokded anymore. + def __init__(self, input_fn, batch_axis, ctx): + """Constructor. Args: - flattened_inputs: Flattened inputs for one each, which should be created - by the `as_sharded_flattened_inputs` API. - - Returns: - A tuple of (`features`, `labels`), where `labels` could be None. - Each one, if present, should have identical structure (single tensor vs - dict) as the one returned by input_fn. + input_fn: input fn for train or eval. + batch_axis: A python tuple of int values describing how each tensor + produced by the Estimator `input_fn` should be split across the TPU + compute shards. + ctx: A `_TPUContext` instance with mode. Raises: - RuntimeError: If the internal data has not been initialized. - ValueError: If the number of expected tensors from `flattened_inputs` - mismatches the recorded structure. + ValueError: If both `sharded_features` and `num_cores` are `None`. """ - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - - expected_num_features = (len(self._feature_names) if self._feature_names - else 1) - if self._has_labels: - expected_num_labels = (len(self._label_names) if self._label_names - else 1) - else: - expected_num_labels = 0 - - expected_num_tensors = expected_num_features + expected_num_labels + self._inputs_structure_recorder = _InputPipeline.InputsStructureRecorder() + + self._sharded_per_core = ctx.is_input_sharded_per_core() + self._input_fn = input_fn + self._infeed_queue = None + self._ctx = ctx + self._batch_axis = batch_axis + + def generate_infeed_enqueue_ops_and_dequeue_fn(self): + """Generates infeed enqueue ops and dequeue_fn.""" + # While tf.while_loop is called, the body function, which invokes + # `enqueue_fn` passed in, is called to construct the graph. So, input_fn + # structure is recorded. + enqueue_ops = self._invoke_input_fn_and_record_structure() + + def dequeue_fn(): + """dequeue_fn is used by TPU to retrieve the tensors.""" + values = self._infeed_queue.generate_dequeue_op() + # The unflatten process uses the structure information recorded above. + return self._inputs_structure_recorder.unflatten_features_and_labels( + values) + + return (enqueue_ops, dequeue_fn) + + def _invoke_input_fn_and_record_structure(self): + if self._sharded_per_core: + # Per-Core input pipeline deployment. + tpu_host_placement_fn = self._ctx.tpu_host_placement_function + enqueue_ops = [] + infeed_queues = [] + + # Invoke input pipeline for each core and placed on the corresponding + # host. + num_hosts = self._ctx.num_hosts + for host_id in range(num_hosts): + host_device = tpu_host_placement_fn(host_id=host_id) + with ops.device(host_device): + with ops.name_scope('input_pipeline_task%d' % (host_id)): + enqueue_ops_fn, infeed_queue_getter = ( + generate_per_core_enqueue_ops_fn_for_host( + self._ctx, self._input_fn, self._inputs_structure_recorder)) + + if _WRAP_INPUT_FN_INTO_WHILE_LOOP: + enqueue_ops.append(_wrap_computation_in_while_loop( + device=host_device, op_fn=enqueue_ops_fn)) + else: + enqueue_ops.append(enqueue_ops_fn()) + # Infeed_queue_getter must be called after enqueue_ops_fn is called. + infeed_queues.append(infeed_queue_getter()) + + # infeed_queue is used to generate dequeue ops. The only thing it uses for + # dequeue is dtypes and types. So, any one can be used. Here, grab the + # first one. + self._infeed_queue = infeed_queues[0] + return enqueue_ops - if expected_num_tensors != len(flattened_inputs): - raise ValueError( - 'The number of flattened tensors mismatches expected num. ' - 'Expected {}, got {}'.format(expected_num_tensors, - len(flattened_inputs))) - if self._feature_names: - unflattened_features = dict(zip(self._feature_names, - flattened_inputs[:expected_num_features])) - else: - # Single tensor case - unflattened_features = flattened_inputs[0] - - if expected_num_labels == 0: - unflattened_label = None - elif self._label_names: - unflattened_label = dict(zip(self._label_names, - flattened_inputs[expected_num_features:])) else: - # Single tensor case. - unflattened_label = flattened_inputs[expected_num_features] - - return unflattened_features, unflattened_label + # TODO(b/67051042): Extend this to multi-host support. + host_id = 0 + host_device = self._ctx.tpu_host_placement_function(host_id=host_id) + def enqueue_fn(): + with ops.device(host_device): + with ops.name_scope('input_pipeline_task%d' % (host_id)): + inputs = self._input_fn() + if isinstance(inputs, tuple): + features, labels = inputs + else: + features, labels = inputs, None + self._inputs_structure_recorder.validate_and_record_structure( + features, labels) + unsharded_tensor_list = ( + self._inputs_structure_recorder.flatten_features_and_labels( + features, labels)) + + self._infeed_queue = tpu_feed.InfeedQueue( + tuple_types=[t.dtype for t in unsharded_tensor_list], + tuple_shapes=[t.shape for t in unsharded_tensor_list], + shard_dimensions=self._batch_axis) + self._infeed_queue.set_number_of_shards(self._ctx.num_cores) + + def placement_fn(core_id): + return self._ctx.tpu_host_placement_function(core_id=core_id) + return ( + self._infeed_queue.split_inputs_and_generate_enqueue_ops( + unsharded_tensor_list, + placement_function=placement_fn)) + + if _WRAP_INPUT_FN_INTO_WHILE_LOOP: + return _wrap_computation_in_while_loop(device=host_device, + op_fn=enqueue_fn) + else: + return enqueue_fn() class _ModelFnWrapper(object): @@ -840,20 +931,17 @@ class _ModelFnWrapper(object): train and eval step. """ - def __init__(self, model_fn, config, params, mode, train_batch_size, - eval_batch_size): + def __init__(self, model_fn, config, params, ctx): self._model_fn = model_fn self._config = config self._params = params - self._mode = mode - self._train_batch_size = train_batch_size - self._eval_batch_size = eval_batch_size + self._ctx = ctx def call_without_tpu(self, features, labels): # Let CrossShardOptimizer be called without TPU in model_fn, since it's # common to set the train_op even when running evaluate() or predict(). with tpu_function.tpu_shard_context(1): - return self._call_model_fn(features, labels, use_tpu=False) + return self._call_model_fn(features, labels) def convert_to_single_tpu_train_step(self, dequeue_fn): """Converts user provided model_fn` as a single train step on TPU. @@ -883,7 +971,7 @@ class _ModelFnWrapper(object): features, labels = dequeue_fn() estimator_spec = self._verify_estimator_spec( - self._call_model_fn(features, labels, use_tpu=True)) + self._call_model_fn(features, labels)) loss, train_op = estimator_spec.loss, estimator_spec.train_op with ops.control_dependencies([train_op]): return array_ops.identity(loss) @@ -915,13 +1003,13 @@ class _ModelFnWrapper(object): A tuple of eval_fn and eval_metrics. The eval_fn representing the eval step for TPU. and eval_metrics is an `_EvalMetrics` instance. """ - eval_metrics = _EvalMetrics() + eval_metrics = _EvalMetrics(self._ctx) def eval_step(total_loss): """Evaluation step function for use inside a while loop.""" features, labels = dequeue_fn() - tpu_estimator_spec = self._call_model_fn(features, labels, use_tpu=True) + tpu_estimator_spec = self._call_model_fn(features, labels) if not isinstance(tpu_estimator_spec, TPUEstimatorSpec): raise RuntimeError( 'estimator_spec used by TPU evaluation must have type' @@ -935,11 +1023,7 @@ class _ModelFnWrapper(object): return math_ops.add(total_loss, loss) return eval_step, eval_metrics - @property - def config(self): - return self._config - - def _call_model_fn(self, features, labels, use_tpu): + def _call_model_fn(self, features, labels): """Calls the model_fn with required parameters.""" model_fn_args = util.fn_args(self._model_fn) kwargs = {} @@ -950,12 +1034,11 @@ class _ModelFnWrapper(object): if 'labels' in model_fn_args: kwargs['labels'] = labels - else: - if labels is not None: - raise ValueError( - 'model_fn does not take labels, but input_fn returns labels.') + elif labels is not None: + raise ValueError( + 'model_fn does not take labels, but input_fn returns labels.') if 'mode' in model_fn_args: - kwargs['mode'] = self._mode + kwargs['mode'] = self._ctx.mode if 'config' in model_fn_args: kwargs['config'] = config if 'params' in model_fn_args: @@ -966,16 +1049,16 @@ class _ModelFnWrapper(object): 'model_fn ({}) does not include params argument, ' 'required by TPUEstimator to pass batch size as ' 'params[\'batch_size\']'.format(self._model_fn)) - if self._mode == model_fn_lib.ModeKeys.TRAIN: - params[_BATCH_SIZE_KEY] = _per_shard_batch_size( - self._train_batch_size, config, use_tpu) - elif (self._mode == model_fn_lib.ModeKeys.EVAL and - self._eval_batch_size is not None): - params[_BATCH_SIZE_KEY] = _per_shard_batch_size( - self._eval_batch_size, config, use_tpu) + + batch_size_for_model_fn = self._ctx.batch_size_for_model_fn + if batch_size_for_model_fn is not None: + params[_BATCH_SIZE_KEY] = batch_size_for_model_fn estimator_spec = self._model_fn(features=features, **kwargs) - if (not use_tpu) and isinstance(estimator_spec, TPUEstimatorSpec): + if (self._ctx.is_running_on_cpu() and + isinstance(estimator_spec, TPUEstimatorSpec)): + # The estimator_spec will be passed to `Estimator` directly, which expects + # type `EstimatorSpec`. return estimator_spec.as_estimator_spec() else: return estimator_spec @@ -998,7 +1081,8 @@ class _ModelFnWrapper(object): class _EvalMetrics(object): """Class wraps TPUEstimator.eval_metrics.""" - def __init__(self): + def __init__(self, ctx): + self._ctx = ctx self._metric_fn = None self._is_dict = False self._tensor_keys = [] @@ -1081,7 +1165,7 @@ class _EvalMetrics(object): raise RuntimeError('Eval metrics have not been recorded yet') return self._tensors - def to_metric_metric_ops_for_tpu(self, run_config, dummy_update_op): + def to_metric_metric_ops_for_tpu(self, dummy_update_op): """Creates the eval_metric_ops now based on the TPU outfeed. `eval_metric_ops` is defined in `EstimatorSpec`. From all shards, tensors @@ -1090,7 +1174,6 @@ class _EvalMetrics(object): metric fn. Args: - run_config: A `RunConfig` instance. dummy_update_op: A dummy update op. Returns: @@ -1102,9 +1185,7 @@ class _EvalMetrics(object): RuntimeError: If outfeed tensor is scalar. """ - num_shards = run_config.tpu_config.num_shards - job = _tpu_job(run_config, model_fn_lib.ModeKeys.EVAL) - job_device = '' if job is None else ('/job:%s' % job) + num_cores = self._ctx.num_cores # For each i, dequeue_ops[i] is a list containing the tensors from all # shards. This list is concatenated later. @@ -1113,8 +1194,9 @@ class _EvalMetrics(object): dequeue_ops.append([]) # Outfeed ops execute on each JF node. - for i in xrange(num_shards): - with ops.device('%s/task:%d/device:TPU:%d' % (job_device, i / 8, i % 8)): + tpu_device_placement_fn = self._ctx.tpu_device_placement_function + for i in xrange(num_cores): + with ops.device(tpu_device_placement_fn(i)): outfeed_tensors = tpu_ops.outfeed_dequeue_tuple( dtypes=self._tensor_dtypes, shapes=self._tensor_shapes) for j, item in enumerate(outfeed_tensors): @@ -1122,7 +1204,7 @@ class _EvalMetrics(object): # It is assumed evaluation always happends on single host TPU system. So, # place all ops on tpu host if possible. - with ops.device('{}/device:CPU:0'.format(job_device)): + with ops.device(self._ctx.tpu_host_placement_function(core_id=0)): for i, item in enumerate(dequeue_ops): if dequeue_ops[i][0].shape.ndims == 0: raise RuntimeError( @@ -1167,9 +1249,9 @@ class TPUEstimator(estimator_lib.Estimator): specify `train_batch_size` in constructor, and then get the batch size for each shard in `input_fn` and `model_fn` by `params['batch_size']`. If `TPUConfig.per_host_input_for_training` is `True`, `input_fn` is invoked per - host rather than per shard. In this case, a global batch size is transformed a + host rather than per core. In this case, a global batch size is transformed a per-host batch size in params for `input_fn`, but `model_fn` still gets - per-shard batch size. + per-core batch size. For evaluation, if `eval_batch_size` is None, it is executed on CPU, even if `use_tpu` is `True`. If `eval_batch_size` is not `None`, it is executed on @@ -1327,9 +1409,7 @@ class TPUEstimator(estimator_lib.Estimator): # We cannot store config and params in this constructor as parent # constructor might change them, such as assigning a temp dir for # config.model_dir. - model_function = _augment_model_fn(model_fn, train_batch_size, - eval_batch_size, use_tpu, - batch_axis) + model_function = self._augment_model_fn(model_fn, batch_axis) # Passing non-None params as wrapped model_fn has it. params = params or {} @@ -1338,12 +1418,13 @@ class TPUEstimator(estimator_lib.Estimator): model_dir=model_dir, config=config, params=params) - self._use_tpu = use_tpu - self._train_batch_size = train_batch_size - self._eval_batch_size = eval_batch_size self._iterations_per_training_loop = ( self._config.tpu_config.iterations_per_loop) + # All properties passed to _TPUContext are immutable. + self._ctx = _TPUContext(self._config, train_batch_size, eval_batch_size, + use_tpu) + def _create_global_step(self, graph): """Creates a global step suitable for TPUs. @@ -1359,10 +1440,10 @@ class TPUEstimator(estimator_lib.Estimator): return _create_global_step(graph) def _convert_train_steps_to_hooks(self, steps, max_steps): - if _is_running_on_cpu(self._use_tpu, model_fn_lib.ModeKeys.TRAIN, - self._eval_batch_size): - return super(TPUEstimator, self)._convert_train_steps_to_hooks( - steps, max_steps) + with self._ctx.with_mode(model_fn_lib.ModeKeys.TRAIN) as ctx: + if ctx.is_running_on_cpu(): + return super(TPUEstimator, self)._convert_train_steps_to_hooks( + steps, max_steps) # On TPU. if steps is None and max_steps is None: @@ -1380,9 +1461,9 @@ class TPUEstimator(estimator_lib.Estimator): steps, max_steps)] def _convert_eval_steps_to_hooks(self, steps): - if _is_running_on_cpu(self._use_tpu, model_fn_lib.ModeKeys.EVAL, - self._eval_batch_size): - return super(TPUEstimator, self)._convert_eval_steps_to_hooks(steps) + with self._ctx.with_mode(model_fn_lib.ModeKeys.EVAL) as ctx: + if ctx.is_running_on_cpu(): + return super(TPUEstimator, self)._convert_eval_steps_to_hooks(steps) if steps is None: raise ValueError('Evaluate `steps` must be set on TPU. Cannot be `None`.') @@ -1422,197 +1503,115 @@ class TPUEstimator(estimator_lib.Estimator): if 'config' in input_fn_args: kwargs['config'] = config - # Setting the batch size in params first. This helps user to have same - # input_fn for use_tpu=True/False. - if mode == model_fn_lib.ModeKeys.TRAIN: - kwargs['params'][_BATCH_SIZE_KEY] = ( - _per_shard_batch_size(self._train_batch_size, config, self._use_tpu) - if not config.tpu_config.per_host_input_for_training else - self._train_batch_size) - elif (mode == model_fn_lib.ModeKeys.EVAL and - self._eval_batch_size is not None): - # For TPU evaluation, input_fn is invoked for one host (instead of shard). - kwargs['params'][_BATCH_SIZE_KEY] = self._eval_batch_size - - if _is_running_on_cpu(self._use_tpu, mode, self._eval_batch_size): - with ops.device('/device:CPU:0'): - return input_fn(**kwargs) - - job = _tpu_job(config, mode) - def placement_function(index): - if job is None: - return '/replica:0/task:0/device:CPU:0' - else: - return '/job:%s/task:%d/device:CPU:0' % (job, index / 8) + with self._ctx.with_mode(mode) as ctx: + # Setting the batch size in params first. This helps user to have same + # input_fn for use_tpu=True/False. + batch_size_for_input_fn = ctx.batch_size_for_input_fn + if batch_size_for_input_fn is not None: + kwargs['params'][_BATCH_SIZE_KEY] = batch_size_for_input_fn - if mode == model_fn_lib.ModeKeys.TRAIN: - if not config.tpu_config.per_host_input_for_training: - # Now for TPU training. - num_shards = config.tpu_config.num_shards - inputs = _InputsHolder(num_shards=num_shards) - for i in range(config.tpu_config.num_shards): - with ops.device(placement_function(i)): - inputs.append_tuple(input_fn(**kwargs)) - return inputs.as_features_and_labels_tuple() - else: - # TODO(xiejw): Extend this to multi-host support. - with ops.device(placement_function(0)): + if ctx.is_running_on_cpu(): + with ops.device('/device:CPU:0'): return input_fn(**kwargs) - # Now for TPU evaluation. - with ops.device(placement_function(0)): - return input_fn(**kwargs) - - -# TODO(b/64607814): Ensure batch_axis works with nested structures. -def _create_infeed_enqueue_ops_and_dequeue_fn(inputs_holder, run_config, - batch_axis, mode): - """Utility to convert input_fn to enqueue and dequeue fns for TPU. - - Args: - inputs_holder: An `_InputsHolder` holding features and labels. - run_config: A `RunConfig` instance. - batch_axis: A python list of batch dimensions. - mode: ModeKeys - - Returns: - A tuple of (dequeue_fn, enqueue_fn) - """ - if inputs_holder.sharded: - sharded_inputs = inputs_holder.as_sharded_flattened_inputs() - - infeed_queue = tpu_feed.InfeedQueue( - number_of_tuple_elements=len(sharded_inputs[0])) - infeed_queue.set_configuration_from_sharded_input_tensors(sharded_inputs) - else: - unsharded_inputs = inputs_holder.as_flattened_inputs() - infeed_queue = tpu_feed.InfeedQueue( - tuple_types=[t.dtype for t in unsharded_inputs], - tuple_shapes=[t.shape for t in unsharded_inputs], - shard_dimensions=batch_axis) - infeed_queue.set_number_of_shards(inputs_holder.num_shards) - - def dequeue_fn(): - """dequeue_fn is used by the train_step in TPU to retrieve the tensors.""" - values = infeed_queue.generate_dequeue_op() - return inputs_holder.unflatten_features_and_labels(values) - - def tpu_ordinal_function(index): - """Return the TPU ordinal associated with a shard. - - Required because the enqueue ops are placed on CPU. - - Args: - index: the shard index - - Returns: - The ordinal of the TPU device the shard's infeed should be placed on. - """ - return index % 8 - - def enqueue_fn(): - """enqueue_fn is used to add ops to the graph to send tensors.""" - if inputs_holder.sharded: - return infeed_queue.generate_enqueue_ops( - sharded_inputs, tpu_ordinal_function=tpu_ordinal_function) - else: - job = _tpu_job(run_config, mode) - def placement_function(index): - if job is None: - return '/replica:0/task:0/device:CPU:0' - else: - # This assumes that if using more than 8 shards, - # the job configuration varies 'task'. - return '/job:%s/task:%d/device:CPU:0' % (job, index / 8) - return infeed_queue.split_inputs_and_generate_enqueue_ops( - unsharded_inputs, placement_function=placement_function) - - return (dequeue_fn, enqueue_fn) - - -def _augment_model_fn(model_fn, train_batch_size, eval_batch_size, use_tpu, - batch_axis): - """Returns a new model_fn, which wraps the TPU support.""" - - def _model_fn(features, labels, mode, config, params): - """A Estimator `model_fn` for TPUEstimator.""" - model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, mode, - train_batch_size, eval_batch_size) - - # TODO(jhseu): Move to PREDICT to TPU. - if _is_running_on_cpu(use_tpu, mode, eval_batch_size): - logging.info('Running %s on CPU', mode) - return model_fn_wrapper.call_without_tpu(features, labels) - - inputs = _InputsHolder(features=features, labels=labels, - num_shards=config.tpu_config.num_shards) - - dequeue_fn, enqueue_fn = _create_infeed_enqueue_ops_and_dequeue_fn( - inputs, config, batch_axis, mode) - - if mode == model_fn_lib.ModeKeys.TRAIN: - loss = _train_on_tpu_system(model_fn_wrapper, dequeue_fn) - hooks = [ - TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn), - training.LoggingTensorHook( - {'loss': array_ops.identity(loss), - 'step': training.get_global_step()}, - every_n_secs=30) - ] - summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss) - with ops.control_dependencies([loss]): - update_ops = _sync_variables_ops() - - # Validate the TPU training graph to catch basic errors - _validate_tpu_training_graph() - - return model_fn_lib.EstimatorSpec( - mode, - loss=loss, - training_hooks=hooks, - train_op=control_flow_ops.group(*update_ops)) - - # Now eval. - total_loss, eval_metric_ops = _eval_on_tpu_system( - model_fn_wrapper, dequeue_fn) - iterations_per_loop_var = _create_iterations_per_loop() - mean_loss = math_ops.div( - total_loss, - math_ops.cast(iterations_per_loop_var, dtype=total_loss.dtype)) - - # Creates a dummy metric update_op for all metrics. Estimator expects all - # metrics in eval_metric_ops have update_op and calls them one by one. The - # real metric update_ops are invoked in a separated thread. So, here give - # Estimator the dummy op for all metrics. - with ops.control_dependencies([mean_loss]): - # After TPU evaluation computation is done (the mean_loss tensor), reads - # all variables back from TPU and updates the eval step counter properly. - internal_ops_to_run = _sync_variables_ops() - internal_ops_to_run.append( - _increase_eval_step_op(iterations_per_loop_var)) - with ops.control_dependencies(internal_ops_to_run): - dummy_update_op = control_flow_ops.no_op() - - eval_metric_ops, eval_update_ops = ( - eval_metric_ops.to_metric_metric_ops_for_tpu( - config, dummy_update_op)) - hooks = [ - TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn, eval_update_ops), - ] - - return model_fn_lib.EstimatorSpec( - mode, - loss=mean_loss, - evaluation_hooks=hooks, - eval_metric_ops=eval_metric_ops) - return _model_fn - - -def _eval_on_tpu_system(model_fn_wrapper, dequeue_fn): + # For TPU computation, input_fn should be invoked in a tf.while_loop for + # performance. While constructing the tf.while_loop, the structure of + # inputs returned by the `input_fn` needs to be recorded. The structure + # includes whether features or labels is dict or single Tensor, dict keys, + # tensor shapes, and dtypes. The recorded structure is used to create the + # infeed dequeue ops, which must be wrapped and passed as a Fn, called + # inside the TPU computation, as the TPU computation is wrapped inside a + # tf.while_loop also. So, we either pass input_fn to model_fn or pass + # dequeue_fn to model_fn. Here, `input_fn` is passed directly as + # `features` in `model_fn` signature. + def _input_fn(): + return input_fn(**kwargs) + return _input_fn + + def _augment_model_fn(self, model_fn, batch_axis): + """Returns a new model_fn, which wraps the TPU support.""" + + def _model_fn(features, labels, mode, config, params): + """A Estimator `model_fn` for TPUEstimator.""" + with self._ctx.with_mode(mode) as ctx: + model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, ctx) + + # TODO(jhseu): Move to PREDICT to TPU. + if ctx.is_running_on_cpu(): + logging.info('Running %s on CPU', mode) + return model_fn_wrapper.call_without_tpu(features, labels) + + assert labels is None, '`labels` passed to `model_fn` must be `None`.' + # TPUEstimator._call_input_fn passes `input_fn` as features to here. + assert callable(features), '`input_fn` is not callable.' + input_fn = features + + input_holders = _InputPipeline(input_fn, batch_axis, ctx) + enqueue_ops, dequeue_fn = ( + input_holders.generate_infeed_enqueue_ops_and_dequeue_fn()) + + if mode == model_fn_lib.ModeKeys.TRAIN: + loss = _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn) + hooks = [ + TPUInfeedOutfeedSessionHook(ctx, enqueue_ops), + training.LoggingTensorHook( + {'loss': array_ops.identity(loss), + 'step': training.get_global_step()}, + every_n_secs=30) + ] + summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss) + with ops.control_dependencies([loss]): + update_ops = _sync_variables_ops() + + # Validate the TPU training graph to catch basic errors + _validate_tpu_training_graph() + + return model_fn_lib.EstimatorSpec( + mode, + loss=loss, + training_hooks=hooks, + train_op=control_flow_ops.group(*update_ops)) + + # Now eval. + total_loss, eval_metric_ops = _eval_on_tpu_system( + ctx, model_fn_wrapper, dequeue_fn) + iterations_per_loop_var = _create_or_get_iterations_per_loop() + mean_loss = math_ops.div( + total_loss, + math_ops.cast(iterations_per_loop_var, dtype=total_loss.dtype)) + + # Creates a dummy metric update_op for all metrics. Estimator expects + # all metrics in eval_metric_ops have update_op and calls them one by + # one. The real metric update_ops are invoked in a separated thread. So, + # here give Estimator the dummy op for all metrics. + with ops.control_dependencies([mean_loss]): + # After TPU evaluation computation is done (the mean_loss tensor), + # reads all variables back from TPU and updates the eval step counter + # properly + internal_ops_to_run = _sync_variables_ops() + internal_ops_to_run.append( + _increase_eval_step_op(iterations_per_loop_var)) + with ops.control_dependencies(internal_ops_to_run): + dummy_update_op = control_flow_ops.no_op() + + eval_metric_ops, eval_update_ops = ( + eval_metric_ops.to_metric_metric_ops_for_tpu(dummy_update_op)) + hooks = [ + TPUInfeedOutfeedSessionHook(ctx, enqueue_ops, eval_update_ops), + ] + + return model_fn_lib.EstimatorSpec( + mode, + loss=mean_loss, + evaluation_hooks=hooks, + eval_metric_ops=eval_metric_ops) + return _model_fn + + +def _eval_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): """Executes `model_fn_wrapper` multiple times on all TPU shards.""" - config = model_fn_wrapper.config.tpu_config - num_shards = config.num_shards - iterations_per_loop_var = _create_iterations_per_loop() + num_cores = ctx.num_cores + iterations_per_loop_var = _create_or_get_iterations_per_loop() single_tpu_eval_step, eval_metric_ops = ( model_fn_wrapper.convert_to_single_tpu_eval_step(dequeue_fn)) @@ -1625,15 +1624,15 @@ def _eval_on_tpu_system(model_fn_wrapper, dequeue_fn): (loss,) = tpu.shard(multi_tpu_eval_steps_on_single_shard, inputs=[], - num_shards=num_shards, + num_shards=num_cores, outputs_from_all_shards=False) return loss, eval_metric_ops -def _train_on_tpu_system(model_fn_wrapper, dequeue_fn): +def _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): """Executes `model_fn_wrapper` multiple times on all TPU shards.""" - num_shards = model_fn_wrapper.config.tpu_config.num_shards - iterations_per_loop_var = _create_iterations_per_loop() + num_cores = ctx.num_cores + iterations_per_loop_var = _create_or_get_iterations_per_loop() single_tpu_train_step = model_fn_wrapper.convert_to_single_tpu_train_step( dequeue_fn) @@ -1647,11 +1646,27 @@ def _train_on_tpu_system(model_fn_wrapper, dequeue_fn): (loss,) = tpu.shard(multi_tpu_train_steps_on_single_shard, inputs=[], - num_shards=num_shards, + num_shards=num_cores, outputs_from_all_shards=False) return loss +def _wrap_computation_in_while_loop(device, op_fn): + """Wraps the ops generated by `op_fn` in tf.while_loop.""" + def computation(i): + with ops.control_dependencies(op_fn()): + return i + 1 + + iterations_per_loop_var = _create_or_get_iterations_per_loop() + # By setting parallel_iterations=1, the parallel execution in while_loop is + # basically turned off. + with ops.device(device): + iterations = array_ops.identity(iterations_per_loop_var) + return control_flow_ops.while_loop( + lambda i: i < iterations, + computation, [constant_op.constant(0)], parallel_iterations=1) + + def _validate_tpu_training_graph(): """Validate graph before running distributed training. -- GitLab From 71bdc0efa737e3094033f0c6ea3779b1fc3c8a94 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 11:34:40 -0700 Subject: [PATCH 206/573] Formatting metric_ops. PiperOrigin-RevId: 172910546 --- .../contrib/metrics/python/ops/metric_ops.py | 591 +++++++++++------- 1 file changed, 382 insertions(+), 209 deletions(-) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 85c8e9038a..09485c4fa2 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -56,7 +56,10 @@ def _safe_div(numerator, denominator, name): name=name) -def _create_local(name, shape, collections=None, validate_shape=True, +def _create_local(name, + shape, + collections=None, + validate_shape=True, dtype=dtypes.float32): """Creates a new local variable. @@ -87,7 +90,9 @@ def _assert_weights_rank(weights, values): return check_ops.assert_rank_in(weights, (0, array_ops.rank(values))) -def _count_condition(values, weights=None, metrics_collections=None, +def _count_condition(values, + weights=None, + metrics_collections=None, updates_collections=None): """Sums the weights of cases where the given values are True. @@ -134,7 +139,9 @@ def _count_condition(values, weights=None, metrics_collections=None, return value_tensor, update_op -def streaming_true_positives(predictions, labels, weights=None, +def streaming_true_positives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -168,12 +175,17 @@ def streaming_true_positives(predictions, labels, weights=None, tuple. """ return metrics.true_positives( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_true_negatives(predictions, labels, weights=None, +def streaming_true_negatives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -206,20 +218,22 @@ def streaming_true_negatives(predictions, labels, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) - is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), - math_ops.equal(predictions, False)) + is_true_negative = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, False)) return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) -def streaming_false_positives(predictions, labels, weights=None, +def streaming_false_positives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -253,12 +267,17 @@ def streaming_false_positives(predictions, labels, weights=None, tuple. """ return metrics.false_positives( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_false_negatives(predictions, labels, weights=None, +def streaming_false_negatives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -291,9 +310,12 @@ def streaming_false_negatives(predictions, labels, weights=None, or tuple. """ return metrics.false_negatives( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) # TODO(ptucker): Move this somewhere common, to share with ops/losses/losses.py. @@ -317,17 +339,18 @@ def _broadcast_weights(weights, values): with ops.name_scope(None, 'broadcast_weights', (values, weights)) as scope: weights_shape = weights.get_shape() values_shape = values.get_shape() - if (weights_shape.is_fully_defined() and - values_shape.is_fully_defined() and + if (weights_shape.is_fully_defined() and values_shape.is_fully_defined() and weights_shape.is_compatible_with(values_shape)): return weights with ops.control_dependencies((_assert_weights_rank(weights, values),)): - return math_ops.multiply( - weights, array_ops.ones_like(values), name=scope) + return math_ops.multiply(weights, array_ops.ones_like(values), name=scope) -def streaming_mean(values, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_mean(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the (weighted) mean of the given values. The `streaming_mean` function creates two local variables, `total` and `count` @@ -365,12 +388,18 @@ def streaming_mean(values, weights=None, metrics_collections=None, or tuple. """ return metrics.mean( - values=values, weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + values=values, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) -def streaming_mean_tensor(values, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_mean_tensor(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `streaming_mean` function which returns a scalar with the @@ -412,12 +441,18 @@ def streaming_mean_tensor(values, weights=None, metrics_collections=None, or tuple. """ return metrics.mean_tensor( - values=values, weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + values=values, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) -def streaming_accuracy(predictions, labels, weights=None, - metrics_collections=None, updates_collections=None, +def streaming_accuracy(predictions, + labels, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Calculates how often `predictions` matches `labels`. @@ -462,13 +497,19 @@ def streaming_accuracy(predictions, labels, weights=None, tuple. """ return metrics.accuracy( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_precision(predictions, labels, weights=None, - metrics_collections=None, updates_collections=None, +def streaming_precision(predictions, + labels, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Computes the precision of the predictions with respect to the labels. @@ -512,13 +553,19 @@ def streaming_precision(predictions, labels, weights=None, tuple. """ return metrics.precision( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_recall(predictions, labels, weights=None, - metrics_collections=None, updates_collections=None, +def streaming_recall(predictions, + labels, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Computes the recall of the predictions with respect to the labels. @@ -560,12 +607,17 @@ def streaming_recall(predictions, labels, weights=None, tuple. """ return metrics.recall( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def _true_negatives(labels, predictions, weights=None, +def _true_negatives(labels, + predictions, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -597,20 +649,22 @@ def _true_negatives(labels, predictions, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) - is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), - math_ops.equal(predictions, False)) + is_true_negative = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, False)) return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) -def streaming_false_positive_rate(predictions, labels, weights=None, +def streaming_false_positive_rate(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -657,30 +711,35 @@ def streaming_false_positive_rate(predictions, labels, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_positive_rate', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_positive_rate', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) false_p, false_positives_update_op = metrics.false_positives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) true_n, true_negatives_update_op = _true_negatives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_fpr(fp, tn, name): return array_ops.where( - math_ops.greater(fp + tn, 0), - math_ops.div(fp, fp + tn), - 0, - name) + math_ops.greater(fp + tn, 0), math_ops.div(fp, fp + tn), 0, name) fpr = compute_fpr(false_p, true_n, 'value') - update_op = compute_fpr( - false_positives_update_op, true_negatives_update_op, 'update_op') + update_op = compute_fpr(false_positives_update_op, true_negatives_update_op, + 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fpr) @@ -691,7 +750,9 @@ def streaming_false_positive_rate(predictions, labels, weights=None, return fpr, update_op -def streaming_false_negative_rate(predictions, labels, weights=None, +def streaming_false_negative_rate(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -738,30 +799,35 @@ def streaming_false_negative_rate(predictions, labels, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_negative_rate', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_negative_rate', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) false_n, false_negatives_update_op = metrics.false_negatives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) true_p, true_positives_update_op = metrics.true_positives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_fnr(fn, tp, name): return array_ops.where( - math_ops.greater(fn + tp, 0), - math_ops.div(fn, fn + tp), - 0, - name) + math_ops.greater(fn + tp, 0), math_ops.div(fn, fn + tp), 0, name) fnr = compute_fnr(false_n, true_p, 'value') - update_op = compute_fnr( - false_negatives_update_op, true_positives_update_op, 'update_op') + update_op = compute_fnr(false_negatives_update_op, true_positives_update_op, + 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fnr) @@ -772,8 +838,11 @@ def streaming_false_negative_rate(predictions, labels, weights=None, return fnr, update_op -def _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights=None, includes=None): +def _streaming_confusion_matrix_at_thresholds(predictions, + labels, + thresholds, + weights=None, + includes=None): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -861,8 +930,8 @@ def _streaming_confusion_matrix_at_thresholds( if weights is not None: broadcast_weights = weights_broadcast_ops.broadcast_weights( math_ops.to_float(weights), predictions) - weights_tiled = array_ops.tile(array_ops.reshape( - broadcast_weights, [1, -1]), [num_thresholds, 1]) + weights_tiled = array_ops.tile( + array_ops.reshape(broadcast_weights, [1, -1]), [num_thresholds, 1]) thresh_tiled.get_shape().assert_is_compatible_with( weights_tiled.get_shape()) else: @@ -877,8 +946,9 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_pos, pred_is_pos)) if weights_tiled is not None: is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add( - true_positives, math_ops.reduce_sum(is_true_positive, 1)) + update_ops['tp'] = state_ops.assign_add(true_positives, + math_ops.reduce_sum( + is_true_positive, 1)) values['tp'] = true_positives if 'fn' in includes: @@ -887,8 +957,9 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_pos, pred_is_neg)) if weights_tiled is not None: is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add( - false_negatives, math_ops.reduce_sum(is_false_negative, 1)) + update_ops['fn'] = state_ops.assign_add(false_negatives, + math_ops.reduce_sum( + is_false_negative, 1)) values['fn'] = false_negatives if 'tn' in includes: @@ -897,8 +968,9 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_neg, pred_is_neg)) if weights_tiled is not None: is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add( - true_negatives, math_ops.reduce_sum(is_true_negative, 1)) + update_ops['tn'] = state_ops.assign_add(true_negatives, + math_ops.reduce_sum( + is_true_negative, 1)) values['tn'] = true_negatives if 'fp' in includes: @@ -907,36 +979,45 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_neg, pred_is_pos)) if weights_tiled is not None: is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add( - false_positives, math_ops.reduce_sum(is_false_positive, 1)) + update_ops['fp'] = state_ops.assign_add(false_positives, + math_ops.reduce_sum( + is_false_positive, 1)) values['fp'] = false_positives return values, update_ops -def streaming_true_positives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_true_positives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('tp',)) return values['tp'], update_ops['tp'] -def streaming_false_negatives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_false_negatives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('fn',)) return values['fn'], update_ops['fn'] -def streaming_false_positives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_false_positives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('fp',)) return values['fp'], update_ops['fp'] -def streaming_true_negatives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_true_negatives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('tn',)) return values['tn'], update_ops['tn'] @@ -996,8 +1077,8 @@ def streaming_curve_points(labels=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope(name, 'curve_points', (labels, predictions, - weights)): + with variable_scope.variable_scope(name, 'curve_points', + (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) kepsilon = 1e-7 # to account for floating point imprecisions @@ -1038,9 +1119,14 @@ def streaming_curve_points(labels=None, return points, update_op -def streaming_auc(predictions, labels, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, - curve='ROC', name=None): +def streaming_auc(predictions, + labels, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + curve='ROC', + name=None): """Computes the approximate AUC via a Riemann sum. The `streaming_auc` function creates four local variables, `true_positives`, @@ -1097,14 +1183,24 @@ def streaming_auc(predictions, labels, weights=None, num_thresholds=200, tuple. """ return metrics.auc( - predictions=predictions, labels=labels, weights=weights, - metrics_collections=metrics_collections, num_thresholds=num_thresholds, - curve=curve, updates_collections=updates_collections, name=name) + predictions=predictions, + labels=labels, + weights=weights, + metrics_collections=metrics_collections, + num_thresholds=num_thresholds, + curve=curve, + updates_collections=updates_collections, + name=name) -def streaming_specificity_at_sensitivity( - predictions, labels, sensitivity, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, name=None): +def streaming_specificity_at_sensitivity(predictions, + labels, + sensitivity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the specificity at a given sensitivity. The `streaming_specificity_at_sensitivity` function creates four local @@ -1154,15 +1250,24 @@ def streaming_specificity_at_sensitivity( or `updates_collections` are not a list or tuple. """ return metrics.specificity_at_sensitivity( - sensitivity=sensitivity, num_thresholds=num_thresholds, - predictions=predictions, labels=labels, weights=weights, + sensitivity=sensitivity, + num_thresholds=num_thresholds, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_sensitivity_at_specificity( - predictions, labels, specificity, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, name=None): +def streaming_sensitivity_at_specificity(predictions, + labels, + specificity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the sensitivity at a given specificity. The `streaming_sensitivity_at_specificity` function creates four local @@ -1212,16 +1317,23 @@ def streaming_sensitivity_at_specificity( or `updates_collections` are not a list or tuple. """ return metrics.sensitivity_at_specificity( - specificity=specificity, num_thresholds=num_thresholds, - predictions=predictions, labels=labels, weights=weights, + specificity=specificity, + num_thresholds=num_thresholds, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_precision_at_thresholds(predictions, labels, thresholds, +def streaming_precision_at_thresholds(predictions, + labels, + thresholds, weights=None, metrics_collections=None, - updates_collections=None, name=None): + updates_collections=None, + name=None): """Computes precision values for different `thresholds` on `predictions`. The `streaming_precision_at_thresholds` function creates four local variables, @@ -1266,14 +1378,21 @@ def streaming_precision_at_thresholds(predictions, labels, thresholds, """ return metrics.precision_at_thresholds( thresholds=thresholds, - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_recall_at_thresholds(predictions, labels, thresholds, - weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_recall_at_thresholds(predictions, + labels, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various recall values for different `thresholds` on `predictions`. The `streaming_recall_at_thresholds` function creates four local variables, @@ -1316,14 +1435,21 @@ def streaming_recall_at_thresholds(predictions, labels, thresholds, """ return metrics.recall_at_thresholds( thresholds=thresholds, - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_false_positive_rate_at_thresholds(predictions, + labels, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various fpr values for different `thresholds` on `predictions`. The `streaming_false_positive_rate_at_thresholds` function creates two @@ -1365,20 +1491,19 @@ def streaming_false_positive_rate_at_thresholds( either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_positive_rate_at_thresholds', - (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_positive_rate_at_thresholds', + (predictions, labels, weights)): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights, includes=('fp', 'tn')) # Avoid division by zero. epsilon = 1e-7 + def compute_fpr(fp, tn, name): return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name) fpr = compute_fpr(values['fp'], values['tn'], 'value') - update_op = compute_fpr( - update_ops['fp'], update_ops['tn'], 'update_op') + update_op = compute_fpr(update_ops['fp'], update_ops['tn'], 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fpr) @@ -1389,9 +1514,13 @@ def streaming_false_positive_rate_at_thresholds( return fpr, update_op -def streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_false_negative_rate_at_thresholds(predictions, + labels, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various fnr values for different `thresholds` on `predictions`. The `streaming_false_negative_rate_at_thresholds` function creates two @@ -1433,20 +1562,19 @@ def streaming_false_negative_rate_at_thresholds( either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_negative_rate_at_thresholds', - (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_negative_rate_at_thresholds', + (predictions, labels, weights)): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights, includes=('fn', 'tp')) # Avoid division by zero. epsilon = 1e-7 + def compute_fnr(fn, tp, name): return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name) fnr = compute_fnr(values['fn'], values['tp'], 'value') - update_op = compute_fnr( - update_ops['fn'], update_ops['tp'], 'update_op') + update_op = compute_fnr(update_ops['fn'], update_ops['tp'], 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fnr) @@ -1469,8 +1597,12 @@ def _at_k_name(name, k=None, class_id=None): @deprecated('2016-11-08', 'Please use `streaming_sparse_recall_at_k`, ' 'and reshape labels from [batch_size] to [batch_size, 1].') -def streaming_recall_at_k(predictions, labels, k, weights=None, - metrics_collections=None, updates_collections=None, +def streaming_recall_at_k(predictions, + labels, + k, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Computes the recall@k of the predictions with respect to dense labels. @@ -1516,11 +1648,8 @@ def streaming_recall_at_k(predictions, labels, k, weights=None, tuple. """ in_top_k = math_ops.to_float(nn.in_top_k(predictions, labels, k)) - return streaming_mean(in_top_k, - weights, - metrics_collections, - updates_collections, - name or _at_k_name('recall', k)) + return streaming_mean(in_top_k, weights, metrics_collections, + updates_collections, name or _at_k_name('recall', k)) # TODO(ptucker): Validate range of values in labels? @@ -1599,10 +1728,14 @@ def streaming_sparse_recall_at_k(predictions, are not a list or tuple. """ return metrics.recall_at_k( - k=k, class_id=class_id, - predictions=predictions, labels=labels, weights=weights, + k=k, + class_id=class_id, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) # TODO(ptucker): Validate range of values in labels? @@ -1684,10 +1817,14 @@ def streaming_sparse_precision_at_k(predictions, are not a list or tuple. """ return metrics.sparse_precision_at_k( - k=k, class_id=class_id, - predictions=predictions, labels=labels, weights=weights, + k=k, + class_id=class_id, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) # TODO(ptucker): Validate range of values in labels? @@ -1766,9 +1903,8 @@ def streaming_sparse_precision_at_top_k(top_k_predictions, ValueError: If `top_k_predictions` has rank < 2. """ default_name = _at_k_name('precision', class_id=class_id) - with ops.name_scope( - name, default_name, - (top_k_predictions, labels, weights)) as name_scope: + with ops.name_scope(name, default_name, + (top_k_predictions, labels, weights)) as name_scope: return metrics_impl._sparse_precision_at_top_k( # pylint: disable=protected-access labels=labels, predictions_idx=top_k_predictions, @@ -1848,8 +1984,8 @@ def sparse_recall_at_top_k(labels, are not a list or tuple. """ default_name = _at_k_name('recall', class_id=class_id) - with ops.name_scope(name, default_name, (top_k_predictions, labels, - weights)) as name_scope: + with ops.name_scope(name, default_name, + (top_k_predictions, labels, weights)) as name_scope: return metrics_impl._sparse_recall_at_top_k( # pylint: disable=protected-access labels=labels, predictions_idx=top_k_predictions, @@ -1919,9 +2055,13 @@ def streaming_sparse_average_precision_at_k(predictions, value matches `metric`. """ return metrics.sparse_average_precision_at_k( - k=k, predictions=predictions, labels=labels, weights=weights, + k=k, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) def streaming_sparse_average_precision_at_top_k(top_k_predictions, @@ -1987,7 +2127,9 @@ def streaming_sparse_average_precision_at_top_k(top_k_predictions, name=name) -def streaming_mean_absolute_error(predictions, labels, weights=None, +def streaming_mean_absolute_error(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2035,12 +2177,18 @@ def streaming_mean_absolute_error(predictions, labels, weights=None, tuple. """ return metrics.mean_absolute_error( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_mean_relative_error(predictions, labels, normalizer, weights=None, +def streaming_mean_relative_error(predictions, + labels, + normalizer, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2089,12 +2237,18 @@ def streaming_mean_relative_error(predictions, labels, normalizer, weights=None, tuple. """ return metrics.mean_relative_error( - normalizer=normalizer, predictions=predictions, labels=labels, - weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + normalizer=normalizer, + predictions=predictions, + labels=labels, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) -def streaming_mean_squared_error(predictions, labels, weights=None, +def streaming_mean_squared_error(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2142,12 +2296,17 @@ def streaming_mean_squared_error(predictions, labels, weights=None, tuple. """ return metrics.mean_squared_error( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_root_mean_squared_error(predictions, labels, weights=None, +def streaming_root_mean_squared_error(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2195,9 +2354,12 @@ def streaming_root_mean_squared_error(predictions, labels, weights=None, tuple. """ return metrics.root_mean_squared_error( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) def streaming_covariance(predictions, @@ -2253,8 +2415,8 @@ def streaming_covariance(predictions, ValueError: If labels and predictions are of different sizes or if either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'covariance', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'covariance', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -2298,22 +2460,22 @@ def streaming_covariance(predictions, # prev_mean_label is E[y_A] in the update equation prev_mean_label = update_mean_label - delta_mean_label - unweighted_batch_coresiduals = ( - (predictions - batch_mean_prediction) * (labels - batch_mean_label)) + unweighted_batch_coresiduals = ((predictions - batch_mean_prediction) * + (labels - batch_mean_label)) # batch_comoment is C_B in the update equation if weights is None: batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals) else: - batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals * - weights) + batch_comoment = math_ops.reduce_sum( + unweighted_batch_coresiduals * weights) # View delta_comoment as = C_AB - C_A in the update equation above. # Since C_A is stored in a var, by how much do we need to increment that var # to make the var = C_AB? - delta_comoment = (batch_comoment + - (prev_mean_prediction - batch_mean_prediction) * - (prev_mean_label - batch_mean_label) * - (prev_count * batch_count / update_count)) + delta_comoment = ( + batch_comoment + (prev_mean_prediction - batch_mean_prediction) * + (prev_mean_label - batch_mean_label) * + (prev_count * batch_count / update_count)) update_comoment = state_ops.assign_add(comoment, delta_comoment) covariance = array_ops.where( @@ -2387,8 +2549,8 @@ def streaming_pearson_correlation(predictions, `weights` is the wrong size, or if either `metrics_collections` or `updates_collections` are not a `list` or `tuple`. """ - with variable_scope.variable_scope( - name, 'pearson_r', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'pearson_r', + (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -2405,13 +2567,14 @@ def streaming_pearson_correlation(predictions, pearson_r = math_ops.truediv( cov, - math_ops.multiply(math_ops.sqrt(var_predictions), - math_ops.sqrt(var_labels)), + math_ops.multiply( + math_ops.sqrt(var_predictions), math_ops.sqrt(var_labels)), name='pearson_r') update_op = math_ops.truediv( update_cov, - math_ops.multiply(math_ops.sqrt(update_var_predictions), - math_ops.sqrt(update_var_labels)), + math_ops.multiply( + math_ops.sqrt(update_var_predictions), + math_ops.sqrt(update_var_labels)), name='update_op') if metrics_collections: @@ -2425,7 +2588,10 @@ def streaming_pearson_correlation(predictions, # TODO(nsilberman): add a 'normalized' flag so that the user can request # normalization if the inputs are not normalized. -def streaming_mean_cosine_distance(predictions, labels, dim, weights=None, +def streaming_mean_cosine_distance(predictions, + labels, + dim, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2471,12 +2637,11 @@ def streaming_mean_cosine_distance(predictions, labels, dim, weights=None, predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) radial_diffs = math_ops.multiply(predictions, labels) - radial_diffs = math_ops.reduce_sum(radial_diffs, - reduction_indices=[dim,], - keep_dims=True) - mean_distance, update_op = streaming_mean(radial_diffs, weights, - None, - None, + radial_diffs = math_ops.reduce_sum( + radial_diffs, reduction_indices=[ + dim, + ], keep_dims=True) + mean_distance, update_op = streaming_mean(radial_diffs, weights, None, None, name or 'mean_cosine_distance') mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -2490,7 +2655,9 @@ def streaming_mean_cosine_distance(predictions, labels, dim, weights=None, return mean_distance, update_op -def streaming_percentage_less(values, threshold, weights=None, +def streaming_percentage_less(values, + threshold, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2530,9 +2697,12 @@ def streaming_percentage_less(values, threshold, weights=None, or tuple. """ return metrics.percentage_below( - values=values, threshold=threshold, weights=weights, + values=values, + threshold=threshold, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) def streaming_mean_iou(predictions, @@ -2584,9 +2754,13 @@ def streaming_mean_iou(predictions, tuple. """ return metrics.mean_iou( - num_classes=num_classes, predictions=predictions, labels=labels, - weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + num_classes=num_classes, + predictions=predictions, + labels=labels, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) def _next_array_size(required_size, growth_factor=1.5): @@ -2601,9 +2775,9 @@ def _next_array_size(required_size, growth_factor=1.5): tf.Tensor with dtype=int32 giving the next array size. """ exponent = math_ops.ceil( - math_ops.log(math_ops.cast(required_size, dtypes.float32)) - / math_ops.log(math_ops.cast(growth_factor, dtypes.float32))) - return math_ops.cast(math_ops.ceil(growth_factor ** exponent), dtypes.int32) + math_ops.log(math_ops.cast(required_size, dtypes.float32)) / math_ops.log( + math_ops.cast(growth_factor, dtypes.float32))) + return math_ops.cast(math_ops.ceil(growth_factor**exponent), dtypes.int32) def streaming_concat(values, @@ -2660,8 +2834,7 @@ def streaming_concat(values, if not 0 <= axis < ndim: raise ValueError('axis = %r not in [0, %r)' % (axis, ndim)) - fixed_shape = [dim.value for n, dim in enumerate(values_shape) - if n != axis] + fixed_shape = [dim.value for n, dim in enumerate(values_shape) if n != axis] if any(value is None for value in fixed_shape): raise ValueError('all dimensions of `values` other than the dimension to ' 'concatenate along must have statically known size') @@ -2804,14 +2977,14 @@ def _remove_squeezable_dimensions(predictions, labels, weights): # Use static rank. if weights_rank - predictions_rank == 1: weights = array_ops.squeeze(weights, [-1]) - elif (weights_rank is None) or ( - weights_shape.dims[-1].is_compatible_with(1)): + elif (weights_rank is + None) or (weights_shape.dims[-1].is_compatible_with(1)): # Use dynamic rank weights = control_flow_ops.cond( - math_ops.equal(array_ops.rank(weights), - math_ops.add(array_ops.rank(predictions), 1)), - lambda: array_ops.squeeze(weights, [-1]), - lambda: weights) + math_ops.equal( + array_ops.rank(weights), + math_ops.add(array_ops.rank(predictions), 1)), + lambda: array_ops.squeeze(weights, [-1]), lambda: weights) return predictions, labels, weights -- GitLab From fafff08cbc3b952d60ee98914c234bb6af09b968 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 11:57:40 -0700 Subject: [PATCH 207/573] Adds the k-MC2 algorithm for efficient seeding of mini batch k-means in TensorFlow. PiperOrigin-RevId: 172914154 --- .../contrib/factorization/g3doc/kmeans.md | 12 +- .../factorization/kernels/clustering_ops.cc | 52 +++++++ .../kernels/clustering_ops_test.cc | 56 ++++++++ .../factorization/ops/clustering_ops.cc | 19 +++ .../kernel_tests/clustering_ops_test.py | 57 ++++++++ .../python/ops/clustering_ops.py | 127 ++++++++++++++++-- 6 files changed, 307 insertions(+), 16 deletions(-) diff --git a/tensorflow/contrib/factorization/g3doc/kmeans.md b/tensorflow/contrib/factorization/g3doc/kmeans.md index b55c9d09ad..c1843f0bf0 100644 --- a/tensorflow/contrib/factorization/g3doc/kmeans.md +++ b/tensorflow/contrib/factorization/g3doc/kmeans.md @@ -24,7 +24,11 @@ the full-batch version. approach for computing the initial cluster assignments that is expensive but is typically less prone to getting stuck in bad local minima. -We provide distributed implementations of both full-batch and mini-batch -K-Means algorithm. Both K-Means++ and random initialization are supported. -The user can also choose between **Cosine** and **Squared Euclidean** distance -metrics. +**[k-MC2](https://www.aaai.org/ocs/index.php/AAAI/AAAI16/paper/view/12147/11759)** +provides a very fast seeding method that provides high quality centers +comparable to K-Means++ seeding. k-MC2 works particularly well if it is combined +with Mini-batch K-Means. + +We provide distributed implementations of both full-batch and mini-batch K-Means +algorithm. K-Means++, k-MC2 and random initialization are supported. The user +can also choose between **Cosine** and **Squared Euclidean** distance metrics. diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc index a2136c08bb..dd61f59585 100644 --- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc +++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc @@ -224,6 +224,58 @@ class KmeansPlusPlusInitializationOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("KmeansPlusPlusInitialization").Device(DEVICE_CPU), KmeansPlusPlusInitializationOp); +// Implementation of one single Markov Chain for the k-MC^2 algorithm +class KMC2ChainInitializationOp : public OpKernel { + public: + explicit KMC2ChainInitializationOp(OpKernelConstruction* context) + : OpKernel(context) { + OP_REQUIRES_OK(context, + context->MatchSignature({DT_FLOAT, DT_INT64}, {DT_INT64})); + } + + void Compute(OpKernelContext* context) override { + const Tensor& distances_tensor = context->input(0); + const Tensor& seed_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsVector(distances_tensor.shape()), + InvalidArgument("Input distances should be a vector.")); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(seed_tensor.shape()), + InvalidArgument("Input seed should be a scalar.")); + const int64 num_points = distances_tensor.dim_size(0); + const int64 seed = seed_tensor.scalar()(); + OP_REQUIRES(context, num_points > 0, + InvalidArgument("Expected distances_tensor.size() > 0.")); + + random::PhiloxRandom random(seed); + random::SimplePhilox rng(&random); + + auto distances = distances_tensor.flat(); + // Set the initial state of the Markov chain to be the first candidate. + int64 selected_index = 0; + float selected_distance = distances(selected_index); + // Build a Markov chain of length num_points. + for (int64 i = 1; i < num_points; ++i) { + const float candidate_distance = distances(i); + // Set the next state of the Markov chain to be the candidate with + // probability min(1, candidate_distance/selected_distance). + if (candidate_distance > rng.RandFloat() * selected_distance) { + selected_index = i; + selected_distance = candidate_distance; + } + } + + Tensor* output_sampled_index_tensor; + OP_REQUIRES_OK(context, + context->allocate_output(0, TensorShape({}), + &output_sampled_index_tensor)); + auto output = output_sampled_index_tensor->scalar(); + // Return the last state of the Markov chain as the new center. + output() = selected_index; + } +}; + +REGISTER_KERNEL_BUILDER(Name("KMC2ChainInitialization").Device(DEVICE_CPU), + KMC2ChainInitializationOp); + // Operator for computing the nearest neighbors for a set of points. class NearestNeighborsOp : public OpKernel { public: diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc b/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc index c4a96b048d..8172a7cebb 100644 --- a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc +++ b/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc @@ -116,6 +116,62 @@ RUN_BM_KmeansPlusPlusInitialization(k3RetriesPerSample); #undef RUN_BM_KmeansPlusPlusInitialization #undef BENCHMARK_KMEANS_PLUS_PLUS +Graph* SetUpKMC2Initialization(int num_points) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor distances(DT_FLOAT, TensorShape({num_points})); + Tensor seed(DT_INT64, TensorShape({})); + distances.flat().setRandom(); + seed.flat().setConstant(12345); + + TF_CHECK_OK( + NodeBuilder("KMC2ChainInitializationOp", "KMC2ChainInitialization") + .Input(test::graph::Constant(g, distances)) + .Input(test::graph::Constant(g, seed)) + .Finalize(g, nullptr /* node */)); + return g; +} + +template +void BM_KMC2Initialization(int iters) { + testing::StopTiming(); + testing::ItemsProcessed(static_cast(iters) * num_points * num_dims * + num_to_sample); + testing::UseRealTime(); + Graph* g = SetUpKMC2Initialization(num_points); + testing::StartTiming(); + test::Benchmark("cpu", g).Run(iters); +} +#define BENCHMARK_KMC2(p, c, d) \ + void BM_KMC2Initialization_##p##_##c##_##d(int iters) { \ + BM_KMC2Initialization(iters); \ + } \ + BENCHMARK(BM_KMC2Initialization_##p##_##c##_##d); + +#define RUN_BM_KMC2Initialization \ + BENCHMARK_KMC2(k10Points, k2Centers, k100Dim); \ + BENCHMARK_KMC2(k10Points, k5Centers, k100Dim); \ + BENCHMARK_KMC2(k10Points, k10Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k10Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k20Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k50Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k200Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k500Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k1kCenters, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k200Centers, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k500Centers, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k1kCenters, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k200Centers, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k500Centers, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k1kCenters, k100Dim) + +RUN_BM_KMC2Initialization; +#undef RUN_BM_KMC2Initialization +#undef BENCHMARK_KMC2 + Graph* SetUpNearestNeighbors(int num_dims, int num_points, int num_centers, int k) { Graph* g = new Graph(OpRegistry::Global()); diff --git a/tensorflow/contrib/factorization/ops/clustering_ops.cc b/tensorflow/contrib/factorization/ops/clustering_ops.cc index f2dfcf7ed0..2686702c1d 100644 --- a/tensorflow/contrib/factorization/ops/clustering_ops.cc +++ b/tensorflow/contrib/factorization/ops/clustering_ops.cc @@ -44,6 +44,25 @@ num_retries_per_sample: Scalar. For each row that is sampled, this parameter samples: Matrix of shape (num_to_sample, d). The sampled rows. )"); +REGISTER_OP("KMC2ChainInitialization") + .Input("distances: float32") + .Input("seed: int64") + .Output("index: int64") + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"( +Returns the index of a data point that should be added to the seed set. + +Entries in distances are assumed to be squared distances of candidate points to +the already sampled centers in the seed set. The op constructs one Markov chain +of the k-MC^2 algorithm and returns the index of one candidate point to be added +as an additional cluster center. + +distances: Vector with squared distances to the closest previously sampled + cluster center for each candidate point. +seed: Scalar. Seed for initializing the random number generator. +index: Scalar with the index of the sampled point. +)"); + REGISTER_OP("NearestNeighbors") .Input("points: float32") .Input("centers: float32") diff --git a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py index 450f64063a..1322f7ce5f 100644 --- a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py +++ b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py @@ -55,6 +55,63 @@ class KmeansPlusPlusInitializationTest(test.TestCase): self.runTestWithSeed(seed) +class KMC2InitializationTest(test.TestCase): + + def runTestWithSeed(self, seed): + with self.test_session(): + distances = np.zeros(1000).astype(np.float32) + distances[6] = 10e7 + distances[4] = 10e3 + + sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed) + self.assertEquals(sampled_point.eval(), 6) + distances[6] = 0.0 + sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed) + self.assertEquals(sampled_point.eval(), 4) + + def testBasic(self): + for seed in range(100): + self.runTestWithSeed(seed) + + +class KMC2InitializationLargeTest(test.TestCase): + + def setUp(self): + self._distances = np.zeros(1001) + self._distances[500] = 100.0 + self._distances[1000] = 50.0 + + def testBasic(self): + with self.test_session(): + counts = {} + seed = 0 + for i in range(50): + sample = clustering_ops.kmc2_chain_initialization( + self._distances, seed + i).eval() + counts[sample] = counts.get(sample, 0) + 1 + self.assertEquals(len(counts), 2) + self.assertTrue(500 in counts) + self.assertTrue(1000 in counts) + self.assertGreaterEqual(counts[500], 5) + self.assertGreaterEqual(counts[1000], 5) + + +class KMC2InitializationCornercaseTest(test.TestCase): + + def setUp(self): + self._distances = np.zeros(10) + + def runTestWithSeed(self, seed): + with self.test_session(): + sampled_point = clustering_ops.kmc2_chain_initialization( + self._distances, seed) + self.assertEquals(sampled_point.eval(), 0) + + def testBasic(self): + for seed in range(100): + self.runTestWithSeed(seed) + + # A simple test that can be verified by hand. class NearestCentersTest(test.TestCase): diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py index d7320aeb3d..96cc80ce24 100644 --- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py +++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py @@ -50,6 +50,7 @@ COSINE_DISTANCE = 'cosine' RANDOM_INIT = 'random' KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus' +KMC2_INIT = 'kmc2' # The name of the variable holding the cluster centers. Used by the Estimator. CLUSTERS_VAR_NAME = 'clusters' @@ -66,7 +67,8 @@ class KMeans(object): use_mini_batch=False, mini_batch_steps_per_iteration=1, random_seed=0, - kmeans_plus_plus_num_retries=2): + kmeans_plus_plus_num_retries=2, + kmc2_chain_length=200): """Creates an object for generating KMeans clustering graph. This class implements the following variants of K-means algorithm: @@ -95,7 +97,8 @@ class KMeans(object): exactly like a full-batch version. Args: - inputs: An input tensor or list of input tensors + inputs: An input tensor or list of input tensors. It is assumed that the + data points have been previously randomly permuted. num_clusters: An integer tensor specifying the number of clusters. This argument is ignored if initial_clusters is a tensor or numpy array. initial_clusters: Specifies the clusters used during initialization. One @@ -104,6 +107,7 @@ class KMeans(object): - a function f(inputs, k) that returns up to k centers from `inputs`. - "random": Choose centers randomly from `inputs`. - "kmeans_plus_plus": Use kmeans++ to choose centers from `inputs`. + - "kmc2": Use the fast k-MC2 algorithm to choose centers from `inputs`. In the last three cases, one batch of `inputs` may not yield `num_clusters` centers, in which case initialization will require multiple batches until enough centers are chosen. In the case of @@ -121,13 +125,17 @@ class KMeans(object): additional points to draw from the current distribution before selecting the best. If a negative value is specified, a heuristic is used to sample O(log(num_to_sample)) additional points. + kmc2_chain_length: Determines how many candidate points are used by the + k-MC2 algorithm to produce one new cluster centers. If a (mini-)batch + contains less points, one new cluster center is generated from the + (mini-)batch. Raises: ValueError: An invalid argument was passed to initial_clusters or distance_metric. """ if isinstance(initial_clusters, str) and initial_clusters not in [ - RANDOM_INIT, KMEANS_PLUS_PLUS_INIT + RANDOM_INIT, KMEANS_PLUS_PLUS_INIT, KMC2_INIT ]: raise ValueError( "Unsupported initialization algorithm '%s'" % initial_clusters) @@ -141,6 +149,7 @@ class KMeans(object): self._mini_batch_steps_per_iteration = int(mini_batch_steps_per_iteration) self._random_seed = random_seed self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries + self._kmc2_chain_length = kmc2_chain_length @classmethod def _distance_graph(cls, inputs, clusters, distance_metric): @@ -302,9 +311,10 @@ class KMeans(object): else: cluster_centers_updated = cluster_centers update_in_steps = None - cluster_counts = (variable_scope.variable( - array_ops.ones([num_clusters], dtype=dtypes.int64)) - if self._use_mini_batch else None) + cluster_counts = ( + variable_scope.variable( + array_ops.ones([num_clusters], dtype=dtypes.int64)) + if self._use_mini_batch else None) return (cluster_centers, cluster_centers_initialized, cluster_counts, cluster_centers_updated, update_in_steps) @@ -359,7 +369,7 @@ class KMeans(object): init_op = _InitializeClustersOpFactory( self._inputs, num_clusters, initial_clusters, self._distance_metric, self._random_seed, self._kmeans_plus_plus_num_retries, - cluster_centers_var, cluster_centers_updated, + self._kmc2_chain_length, cluster_centers_var, cluster_centers_updated, cluster_centers_initialized).op() cluster_centers = cluster_centers_var @@ -520,8 +530,9 @@ class KMeans(object): array_ops.reshape(array_ops.shape(inp)[0], [-1])), [-1, 1]), cluster_idx, num_clusters)) with ops.colocate_with(cluster_centers, ignore_existing=True): - new_clusters_centers = math_ops.add_n(cluster_sums) / (math_ops.cast( - math_ops.add_n(cluster_counts), cluster_sums[0].dtype) + epsilon) + new_clusters_centers = math_ops.add_n(cluster_sums) / ( + math_ops.cast(math_ops.add_n(cluster_counts), cluster_sums[0].dtype) + + epsilon) if self._clusters_l2_normalized(): new_clusters_centers = nn_impl.l2_normalize(new_clusters_centers, dim=1) return state_ops.assign(cluster_centers, new_clusters_centers) @@ -548,9 +559,12 @@ class _InitializeClustersOpFactory(object): cluster_centers_initialized := true """ + # TODO(ccolby): Refactor this class so that kmc2 isn't so much a special case. + def __init__(self, inputs, num_clusters, initial_clusters, distance_metric, - random_seed, kmeans_plus_plus_num_retries, cluster_centers, - cluster_centers_updated, cluster_centers_initialized): + random_seed, kmeans_plus_plus_num_retries, kmc2_chain_length, + cluster_centers, cluster_centers_updated, + cluster_centers_initialized): """Creates an op factory. Args: @@ -560,6 +574,7 @@ class _InitializeClustersOpFactory(object): distance_metric: See KMeans constructor. random_seed: See KMeans constructor. kmeans_plus_plus_num_retries: See KMeans constructor. + kmc2_chain_length: See KMeans constructor. cluster_centers: The TF variable holding the initial centers. It may already contain some centers when the op is executed. cluster_centers_updated: A second TF variable to hold a copy of the @@ -575,6 +590,7 @@ class _InitializeClustersOpFactory(object): self._distance_metric = distance_metric self._random_seed = random_seed self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries + self._kmc2_chain_length = kmc2_chain_length self._cluster_centers = cluster_centers self._cluster_centers_updated = cluster_centers_updated self._cluster_centers_initialized = cluster_centers_initialized @@ -604,6 +620,90 @@ class _InitializeClustersOpFactory(object): math_ops.to_int64(self._num_remaining), self._random_seed, self._kmeans_plus_plus_num_retries) + def _kmc2_multiple_centers(self): + """Adds new initial cluster centers using the k-MC2 algorithm. + + In each call to the op, the provided batch is split into subsets based on + the specified `kmc2_chain_length`. On each subset, a single Markov chain of + the k-MC2 algorithm is used to add *one* new center cluster center. If there + are less than `kmc2_chain_length` points in the subset, a single center is + added using one Markov chain on the full input. It is assumed that the + provided batch has previously been randomly permuted. Otherwise, k-MC2 may + return suboptimal centers. + + Returns: + An op that adds new cluster centers. + """ + # The op only operates on the first shard of data. + first_shard = self._inputs[0] + # Number of points in the input that can be used. + batch_size = array_ops.shape(first_shard)[0] + # Maximum number of subsets such that the size of each subset is at least + # `kmc2_chain_length`. Final subsets may be larger. + max_to_sample = math_ops.cast( + batch_size / self._kmc2_chain_length, dtype=dtypes.int32) + # We sample at least one new center and at most all remaining centers. + num_to_sample = math_ops.maximum( + math_ops.minimum(self._num_remaining, max_to_sample), 1) + + def _cond(i, _): + """Stopping condition for the while loop.""" + return math_ops.less(i, num_to_sample) + + def _body(i, _): + """Body that adds a single new center based on a subset.""" + + def _sample_random(): + """Returns a random point as a cluster center.""" + # By assumption the batch is reshuffled and _sample_random is always + # called for i=0. Hence, we simply return the first point. + new_center = array_ops.reshape(first_shard[0], [1, -1]) + if self._distance_metric == COSINE_DISTANCE: + new_center = nn_impl.l2_normalize(new_center, dim=1) + return new_center + + def _sample_kmc2_chain(): + """Returns previous centers as well as a new center sampled using k-MC2. + """ + # Extract the subset from the underlying batch. + start = i * self._kmc2_chain_length + end = start + self._kmc2_chain_length + subset = first_shard[start:end] + # Compute the distances from points in the subset to previous centers. + _, distances = gen_clustering_ops.nearest_neighbors( + subset, self._cluster_centers, 1) + # Sample index of new center using k-MC2 Markov chain. + new_center_index = gen_clustering_ops.kmc2_chain_initialization( + array_ops.squeeze(distances), self._random_seed) + # Extract actual new center. + newly_sampled_center = array_ops.reshape(subset[new_center_index], + [1, -1]) + # Return concatenation with previously sampled centers. + if self._distance_metric == COSINE_DISTANCE: + newly_sampled_center = nn_impl.l2_normalize( + newly_sampled_center, dim=1) + return array_ops.concat([self._cluster_centers, newly_sampled_center], + 0) + + # Obtain a random point if there are no previously sampled centers. + # Otherwise, construct a k-MC2 Markov chain. + new_centers = control_flow_ops.cond( + math_ops.equal(self._num_selected, 0), _sample_random, + _sample_kmc2_chain) + # Assign new cluster centers to underlying variable. + assigned_centers = state_ops.assign( + self._cluster_centers, new_centers, validate_shape=False) + if self._cluster_centers_updated is not self._cluster_centers: + assigned_centers = state_ops.assign( + self._cluster_centers_updated, + assigned_centers, + validate_shape=False) + return i + 1, self._num_clusters - array_ops.shape(assigned_centers)[0] + + # Add num_to_sample new data points. + _, num_remaining = control_flow_ops.while_loop(_cond, _body, [0, 0]) + return num_remaining + def _greedy_batch_sampler(self, sampler): # If the input dataset size is smaller than the number of centers # remaining, choose the entire input dataset as centers. This can happen @@ -657,7 +757,10 @@ class _InitializeClustersOpFactory(object): with ops.control_dependencies([ check_ops.assert_positive(self._num_remaining), ]): - num_now_remaining = self._add_new_centers() + if self._initial_clusters == KMC2_INIT: + num_now_remaining = self._kmc2_multiple_centers() + else: + num_now_remaining = self._add_new_centers() return control_flow_ops.cond( math_ops.equal(num_now_remaining, 0), lambda: state_ops.assign(self._cluster_centers_initialized, True), -- GitLab From 58bdae2f8b7bead45537092f39f6fa6fd15c50d0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 12:08:44 -0700 Subject: [PATCH 208/573] Work around for compiler bug in GCC on Android. PiperOrigin-RevId: 172915900 --- tensorflow/core/kernels/transpose_functor.h | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h index 9781fe3b61..add4635331 100644 --- a/tensorflow/core/kernels/transpose_functor.h +++ b/tensorflow/core/kernels/transpose_functor.h @@ -201,17 +201,26 @@ Status DoTransposeImpl(const Device& d, const Tensor& in, case DT_COMPLEX64: if (conjugate) { - Transpose::run(d, in, perm, out); +#if defined(__ANDROID__) and !defined(__clang__) + // Workaround for GCC compiler bug in Android toolchain. + return errors::Unimplemented( + "Conjugate transpose of complex64 not supported for GCC on " + "Android."); +#else + Transpose::run(d, in, perm, out); +#endif } else { - Transpose::run(d, in, perm, out); + Transpose::run(d, in, perm, out); } break; case DT_COMPLEX128: if (conjugate) { - Transpose::run(d, in, perm, out); + Transpose::run(d, in, perm, + out); } else { - Transpose::run(d, in, perm, out); + Transpose::run(d, in, perm, + out); } break; -- GitLab From b68a3f2e445cdc749f380387b910f6eac72e5dcf Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Fri, 20 Oct 2017 12:26:09 -0700 Subject: [PATCH 209/573] Iterating through a map in protobuf is essentially nondeterministic. This CL enables us to traverse the map in a deterministic order by sorting the keys first. PiperOrigin-RevId: 172918084 --- tensorflow/compiler/xla/service/user_computation.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index b3506b72bf..065d2580c6 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include #include +#include #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" @@ -1843,10 +1844,17 @@ UserComputation::GetEmbeddedComputations( XLA_VLOG_LINES(3, session_computation_.DebugString()); std::vector computations; + std::vector sorted_handles; for (const auto& handle_request : session_computation_.requests()) { - int64 handle_value = handle_request.first; + sorted_handles.push_back(handle_request.first); + } + std::sort(sorted_handles.begin(), sorted_handles.end()); + for (int64 handle : sorted_handles) { + const auto& handle_request = session_computation_.requests().find(handle); + CHECK(handle_request != session_computation_.requests().end()); + int64 handle_value = handle_request->first; if (handle_value <= version) { - const OperationRequest& request = handle_request.second; + const OperationRequest& request = handle_request->second; switch (request.request().op_case()) { case OpRequest::kCallRequest: { CHECK_EQ(1, request.embedded_computation_versions_size()); -- GitLab From aada11e19a1ceb901f490aa3c064f2778cb2acf2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 12:46:29 -0700 Subject: [PATCH 210/573] Exposes the read_batch_size argument to read_batch_features. PiperOrigin-RevId: 172920603 --- .../learn/python/learn/learn_io/graph_io.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py index bdb88b89bb..4b34fc6284 100644 --- a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py +++ b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py @@ -442,7 +442,8 @@ def read_keyed_batch_features(file_pattern, feature_queue_capacity=100, num_enqueue_threads=2, parse_fn=None, - name=None): + name=None, + read_batch_size=None): """Adds operations to read, queue, batch and parse `Example` protos. Given file pattern (or list of files), will setup a queue for file names, @@ -482,6 +483,8 @@ def read_keyed_batch_features(file_pattern, parse_fn: Parsing function, takes `Example` Tensor returns parsed representation. If `None`, no parsing is done. name: Name of resulting op. + read_batch_size: An int or scalar `Tensor` specifying the number of + records to read at once. If `None`, defaults to `batch_size`. Returns: Returns tuple of: @@ -493,6 +496,7 @@ def read_keyed_batch_features(file_pattern, """ with ops.name_scope(name, 'read_batch_features', [file_pattern]) as scope: + if read_batch_size is None: read_batch_size = batch_size keys, examples = read_keyed_batch_examples( file_pattern, batch_size, @@ -501,7 +505,7 @@ def read_keyed_batch_features(file_pattern, num_epochs=num_epochs, queue_capacity=queue_capacity, num_threads=reader_num_threads, - read_batch_size=batch_size, + read_batch_size=read_batch_size, parse_fn=parse_fn, name=scope) # Parse the example. @@ -727,7 +731,8 @@ def read_batch_features(file_pattern, reader_num_threads=1, num_enqueue_threads=2, parse_fn=None, - name=None): + name=None, + read_batch_size=None): """Adds operations to read, queue, batch and parse `Example` protos. Given file pattern (or list of files), will setup a queue for file names, @@ -768,6 +773,8 @@ def read_batch_features(file_pattern, parse_fn: Parsing function, takes `Example` Tensor returns parsed representation. If `None`, no parsing is done. name: Name of resulting op. + read_batch_size: An int or scalar `Tensor` specifying the number of + records to read at once. If `None`, defaults to `batch_size`. Returns: A dict of `Tensor` or `SparseTensor` objects for each in `features`. @@ -786,6 +793,7 @@ def read_batch_features(file_pattern, reader_num_threads=reader_num_threads, feature_queue_capacity=feature_queue_capacity, num_enqueue_threads=num_enqueue_threads, + read_batch_size=read_batch_size, parse_fn=parse_fn, name=name) return features -- GitLab From 5c331cfd573984287778aab02794dd86ba1f3006 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 12:47:57 -0700 Subject: [PATCH 211/573] The new array class provides a way to simplify the implementation of these classes by eliminating a large number of duplicated code. Removing the old API is non-trivial because of the existing users outside of tensorflow. PiperOrigin-RevId: 172920837 --- .../compiler/xla/client/computation_builder.h | 41 +++--- tensorflow/compiler/xla/layout_util.cc | 4 + tensorflow/compiler/xla/layout_util.h | 1 + tensorflow/compiler/xla/literal_util.h | 121 ++++++++---------- 4 files changed, 85 insertions(+), 82 deletions(-) diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h index cdd9c8847f..93c2a80678 100644 --- a/tensorflow/compiler/xla/client/computation_builder.h +++ b/tensorflow/compiler/xla/client/computation_builder.h @@ -138,6 +138,11 @@ class ComputationBuilder { ComputationDataHandle ConstantR2( std::initializer_list> values); template + ComputationDataHandle ConstantFromArrayWithLayout( + const Array& values, const Layout& layout); + template + ComputationDataHandle ConstantFromArray(const Array& values); + template ComputationDataHandle ConstantR2FromArray2DWithLayout( const Array2D& values, const Layout& layout); template @@ -910,48 +915,54 @@ ComputationDataHandle ComputationBuilder::ConstantR2( } template -ComputationDataHandle ComputationBuilder::ConstantR2FromArray2DWithLayout( - const Array2D& values, const Layout& layout) { +ComputationDataHandle ComputationBuilder::ConstantFromArrayWithLayout( + const Array& values, const Layout& layout) { return ConstantOp([&values, &layout](Literal* literal) { - literal->PopulateR2FromArray2DWithLayout(values, layout); + literal->PopulateFromArrayWithLayout(values, layout); }); } +template +ComputationDataHandle ComputationBuilder::ConstantFromArray( + const Array& values) { + return ConstantOp( + [&values](Literal* literal) { literal->PopulateFromArray(values); }); +} + +template +ComputationDataHandle ComputationBuilder::ConstantR2FromArray2DWithLayout( + const Array2D& values, const Layout& layout) { + return ConstantFromArrayWithLayout(values, layout); +} + template ComputationDataHandle ComputationBuilder::ConstantR2FromArray2D( const Array2D& values) { - return ConstantOp( - [&values](Literal* literal) { literal->PopulateR2FromArray2D(values); }); + return ConstantFromArray(values); } template ComputationDataHandle ComputationBuilder::ConstantR3FromArray3DWithLayout( const Array3D& values, const Layout& layout) { - return ConstantOp([&values, &layout](Literal* literal) { - literal->PopulateR3FromArray3DWithLayout(values, layout); - }); + return ConstantFromArrayWithLayout(values, layout); } template ComputationDataHandle ComputationBuilder::ConstantR3FromArray3D( const Array3D& values) { - return ConstantOp( - [&values](Literal* literal) { literal->PopulateR3FromArray3D(values); }); + return ConstantFromArray(values); } template ComputationDataHandle ComputationBuilder::ConstantR4FromArray4DWithLayout( const Array4D& values, const Layout& layout) { - return ConstantOp([&values, &layout](Literal* literal) { - literal->PopulateR4FromArray4DWithLayout(values, layout); - }); + return ConstantFromArrayWithLayout(values, layout); } template ComputationDataHandle ComputationBuilder::ConstantR4FromArray4D( const Array4D& values) { - return ConstantOp( - [&values](Literal* literal) { literal->PopulateR4FromArray4D(values); }); + return ConstantFromArray(values); } } // namespace xla diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc index 011fc3c194..5c2cc2a7a9 100644 --- a/tensorflow/compiler/xla/layout_util.cc +++ b/tensorflow/compiler/xla/layout_util.cc @@ -83,6 +83,10 @@ Layout CreateDefaultLayoutForRank(int64 rank) { return CreateDefaultLayoutForRank(shape.dimensions_size()); } +/* static */ Layout LayoutUtil::GetDefaultLayoutForRank(int64 rank) { + return CreateDefaultLayoutForRank(rank); +} + /* static */ Layout LayoutUtil::GetDefaultLayoutForR2() { return CreateDefaultLayoutForRank(2); } diff --git a/tensorflow/compiler/xla/layout_util.h b/tensorflow/compiler/xla/layout_util.h index 5de0a653f6..bc42e22229 100644 --- a/tensorflow/compiler/xla/layout_util.h +++ b/tensorflow/compiler/xla/layout_util.h @@ -40,6 +40,7 @@ class LayoutUtil { static Layout GetDefaultLayoutForShape(const Shape& shape); // Helper functions that create default layouts for various ranks. + static Layout GetDefaultLayoutForRank(int64 rank); static Layout GetDefaultLayoutForR2(); static Layout GetDefaultLayoutForR3(); static Layout GetDefaultLayoutForR4(); diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h index e8cee732d4..4063cb05a9 100644 --- a/tensorflow/compiler/xla/literal_util.h +++ b/tensorflow/compiler/xla/literal_util.h @@ -334,6 +334,11 @@ class Literal { // WithLayout use the default XLA layout for the literal's linear // representation in memory. template + static std::unique_ptr CreateFromArray(const Array& values); + template + static std::unique_ptr CreateFromArrayWithLayout( + const Array& values, const Layout& layout); + template static std::unique_ptr CreateR2FromArray2D( const Array2D& values); template @@ -481,6 +486,11 @@ class Literal { std::initializer_list> values, const Layout& layout); template + void PopulateFromArray(const Array& values); + template + void PopulateFromArrayWithLayout(const Array& values, + const Layout& layout); + template void PopulateR2FromArray2D(const Array2D& values); template void PopulateR2FromArray2DWithLayout(const Array2D& values, @@ -816,33 +826,42 @@ template } template -/* static */ std::unique_ptr Literal::CreateR2FromArray2DWithLayout( - const Array2D& values, const Layout& layout) { +/* static */ std::unique_ptr Literal::CreateFromArrayWithLayout( + const Array& values, const Layout& layout) { auto literal = MakeUnique(); - literal->PopulateR2FromArray2DWithLayout(values, layout); + literal->PopulateFromArrayWithLayout(values, layout); return literal; } +template +/* static */ std::unique_ptr Literal::CreateFromArray( + const Array& values) { + return CreateFromArrayWithLayout( + values, LayoutUtil::GetDefaultLayoutForRank(values.num_dimensions())); +} + +template +/* static */ std::unique_ptr Literal::CreateR2FromArray2DWithLayout( + const Array2D& values, const Layout& layout) { + return CreateFromArrayWithLayout(values, layout); +} + template /* static */ std::unique_ptr Literal::CreateR2FromArray2D( const Array2D& values) { - return CreateR2FromArray2DWithLayout(values, - LayoutUtil::GetDefaultLayoutForR2()); + return CreateFromArray(values); } template /* static */ std::unique_ptr Literal::CreateR3FromArray3DWithLayout( const Array3D& values, const Layout& layout) { - auto literal = MakeUnique(); - literal->PopulateR3FromArray3DWithLayout(values, layout); - return literal; + return CreateFromArrayWithLayout(values, layout); } template /* static */ std::unique_ptr Literal::CreateR3FromArray3D( const Array3D& values) { - return CreateR3FromArray3DWithLayout(values, - LayoutUtil::GetDefaultLayoutForR3()); + return CreateFromArray(values); } template @@ -901,16 +920,13 @@ template template /* static */ std::unique_ptr Literal::CreateR4FromArray4D( const Array4D& values) { - return CreateR4FromArray4DWithLayout(values, - LayoutUtil::GetDefaultLayoutForR4()); + return CreateFromArray(values); } template /* static */ std::unique_ptr Literal::CreateR4FromArray4DWithLayout( const Array4D& values, const Layout& layout) { - auto literal = MakeUnique(); - literal->PopulateR4FromArray4DWithLayout(values, layout); - return literal; + return CreateFromArrayWithLayout(values, layout); } template @@ -1070,82 +1086,53 @@ void Literal::PopulateR2( } template -void Literal::PopulateR2FromArray2DWithLayout(const Array2D& values, - const Layout& layout) { +void Literal::PopulateFromArrayWithLayout(const Array& values, + const Layout& layout) { *mutable_shape() = ShapeUtil::MakeShapeWithLayout( - primitive_util::NativeToPrimitiveType(), - {values.height(), values.width()}, AsInt64Slice(layout.minor_to_major())); + primitive_util::NativeToPrimitiveType(), values.dimensions(), + AsInt64Slice(layout.minor_to_major())); + Reserve(values.num_elements()); + values.Each([this](tensorflow::gtl::ArraySlice indices, + NativeT value) { this->Set(indices, value); }); +} - const int64 dim1_size = values.width(); - const int64 dim0_size = values.height(); - CHECK_EQ(dim0_size, shape().dimensions(0)); - CHECK_EQ(dim1_size, shape().dimensions(1)); - Reserve(dim1_size * dim0_size); - for (int64 dim0 = 0; dim0 < dim0_size; ++dim0) { - for (int64 dim1 = 0; dim1 < dim1_size; ++dim1) { - Set({dim0, dim1}, values(dim0, dim1)); - } - } +template +void Literal::PopulateFromArray(const Array& values) { + PopulateFromArrayWithLayout( + values, LayoutUtil::GetDefaultLayoutForRank(values.num_dimensions())); +} + +template +void Literal::PopulateR2FromArray2DWithLayout(const Array2D& values, + const Layout& layout) { + PopulateFromArrayWithLayout(values, layout); } template void Literal::PopulateR2FromArray2D(const Array2D& values) { - PopulateR2FromArray2DWithLayout(values, LayoutUtil::GetDefaultLayoutForR2()); + PopulateFromArray(values); } template void Literal::PopulateR3FromArray3DWithLayout(const Array3D& values, const Layout& layout) { - *mutable_shape() = ShapeUtil::MakeShapeWithLayout( - primitive_util::NativeToPrimitiveType(), - {values.n1(), values.n2(), values.n3()}, - AsInt64Slice(layout.minor_to_major())); - - CHECK_EQ(values.n1(), shape().dimensions(0)); - CHECK_EQ(values.n2(), shape().dimensions(1)); - CHECK_EQ(values.n3(), shape().dimensions(2)); - Reserve(values.n1() * values.n2() * values.n3()); - for (int64 dim0 = 0; dim0 < values.n1(); ++dim0) { - for (int64 dim1 = 0; dim1 < values.n2(); ++dim1) { - for (int64 dim2 = 0; dim2 < values.n3(); ++dim2) { - Set({dim0, dim1, dim2}, values(dim0, dim1, dim2)); - } - } - } + PopulateFromArrayWithLayout(values, layout); } template void Literal::PopulateR3FromArray3D(const Array3D& values) { - PopulateR3FromArray3DWithLayout(values, LayoutUtil::GetDefaultLayoutForR3()); + PopulateFromArray(values); } template void Literal::PopulateR4FromArray4DWithLayout(const Array4D& values, const Layout& layout) { - *mutable_shape() = ShapeUtil::MakeShapeWithLayout( - primitive_util::NativeToPrimitiveType(), - {values.planes(), values.depth(), values.height(), values.width()}, - AsInt64Slice(layout.minor_to_major())); - - CHECK_EQ(values.n1(), shape().dimensions(0)); - CHECK_EQ(values.n2(), shape().dimensions(1)); - CHECK_EQ(values.n3(), shape().dimensions(2)); - CHECK_EQ(values.n4(), shape().dimensions(3)); - Reserve(values.n1() * values.n2() * values.n3() * values.n4()); - for (int64 dim0 = 0; dim0 < values.n1(); ++dim0) { - for (int64 dim1 = 0; dim1 < values.n2(); ++dim1) { - for (int64 dim2 = 0; dim2 < values.n3(); ++dim2) { - for (int64 dim3 = 0; dim3 < values.n4(); ++dim3) { - Set({dim0, dim1, dim2, dim3}, values(dim0, dim1, dim2, dim3)); - } - } - } - } + PopulateFromArrayWithLayout(values, layout); } template void Literal::PopulateR4FromArray4D(const Array4D& values) { - PopulateR4FromArray4DWithLayout(values, LayoutUtil::GetDefaultLayoutForR4()); + PopulateFromArray(values); } template -- GitLab From 5bb971864220e0afdb5587680f444d3779a0f2cf Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Fri, 20 Oct 2017 12:56:51 -0700 Subject: [PATCH 212/573] TFE: Raises an error when attempting to save multiple ResourceVariable objects with the same shared_name. The only way to get multiple objects is if they're created in different Graphs/IsolateTest contexts. Previously this snuck by because of a list -> dictionary conversion without key checking. Allows the same object to be passed multiple times (so people don't need to de-duplicate their lists). PiperOrigin-RevId: 172921932 --- tensorflow/contrib/eager/python/saver_test.py | 34 +++++++++++++++++++ tensorflow/python/training/saver.py | 9 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py index c89554e6dd..1605435d8d 100644 --- a/tensorflow/contrib/eager/python/saver_test.py +++ b/tensorflow/contrib/eager/python/saver_test.py @@ -54,6 +54,40 @@ class SaverTest(test.TestCase): saver.restore(ckpt_prefix) self.assertEqual(v1.read_value().numpy(), 1.0) + def testSameNameNoClobbering(self): + with context.eager_mode(), ops.device(self._dev()): + # Note that this test purposefully uses Graphs rather than + # IsolateTest. Users are more likely to accidentally create the same + # variable name this way. + first_graph = ops.Graph() + with first_graph.as_default(): + v1_first_graph = resource_variable_ops.ResourceVariable(1.0, name='v1') + with ops.Graph().as_default(): + v1_second_graph = resource_variable_ops.ResourceVariable(2.0, name='v1') + saver = _saver.Saver([v1_first_graph, v1_second_graph]) + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + with self.assertRaisesRegexp(ValueError, 'v1'): + saver.save(ckpt_prefix) + + def testDifferentGraphError(self): + with context.eager_mode(), ops.device(self._dev()): + with ops.Graph().as_default(): + v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') + with ops.Graph().as_default(): + saver = _saver.Saver([v1]) + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + with self.assertRaisesRegexp(ValueError, 'Graph'): + saver.save(ckpt_prefix) + + def testSameObjectOK(self): + with context.eager_mode(), ops.device(self._dev()): + v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') + # While different objects with the same shared_name are not good, passing + # in the same object multiple times is fine. + saver = _saver.Saver([v1, v1]) + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + saver.save(ckpt_prefix) + def testRestoreOnCreate(self): with ops.device(self._dev()): def model(init_val): diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index b1926f4eaf..c4c1df22eb 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -557,7 +557,14 @@ class BaseSaverBuilder(object): if not isinstance(var, resource_variable_ops.ResourceVariable): raise ValueError("Can only save/restore ResourceVariable eager " "mode is enabled, type: %s." % type(var)) - names_to_saveables[var._shared_name] = var + set_var = names_to_saveables.setdefault(var._shared_name, var) + if set_var is not var: + raise ValueError( + ("Two different ResourceVariable objects with the same " + "shared_name '%s' were passed to the Saver. This likely means " + "that they were created in different Graphs or isolation " + "contexts, and may not be checkpointed together.") % ( + var._shared_name,)) # pylint: enable=protected-access return names_to_saveables -- GitLab From 54503483ef987c6488d7bc2bd3c4b1d34fbd1f26 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 20 Oct 2017 13:01:41 -0700 Subject: [PATCH 213/573] Enables silent copies of eager tensors for specially-constructed contexts. PiperOrigin-RevId: 172922467 --- tensorflow/c/eager/BUILD | 3 +- tensorflow/c/eager/c_api.cc | 64 +++++++++++++++++++++++++---- tensorflow/c/eager/c_api.h | 16 ++++++++ tensorflow/c/eager/c_api_internal.h | 3 ++ tensorflow/c/eager/c_api_test.cc | 46 +++++++++++++++++++++ 5 files changed, 122 insertions(+), 10 deletions(-) diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index 96f3c3e195..c77896b80b 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -3,6 +3,7 @@ licenses(["notice"]) # Apache 2.0 load( "//tensorflow:tensorflow.bzl", + "tf_cuda_cc_test", "tf_cc_test", "tf_copts", "tf_cuda_library", @@ -50,7 +51,7 @@ tf_cuda_library( ], ) -tf_cc_test( +tf_cuda_cc_test( name = "c_api_test", srcs = ["c_api_test.cc"], deps = [ diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 334c02bff9..28ea2edee4 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -61,6 +61,11 @@ void TFE_ContextOptionsSetConfig(TFE_ContextOptions* options, const void* proto, TF_SetConfig(&options->session_options, proto, proto_len, status); } +void TFE_ContextOptionsSetDevicePlacementPolicy( + TFE_ContextOptions* options, TFE_ContextDevicePlacementPolicy policy) { + options->policy = policy; +} + void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; } TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { @@ -80,6 +85,7 @@ TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { } TFE_Context* ret = new TFE_Context(session); + ret->policy = opts->policy; ret->pflr.reset(new tensorflow::ProcessFunctionLibraryRuntime( ret->session->device_mgr, opts->session_options.options.env, TF_GRAPH_DEF_VERSION, &ret->func_lib_def, {})); @@ -417,8 +423,10 @@ void TFE_OpSetAttrShapeList(TFE_Op* op, const char* attr_name, namespace { tensorflow::Status ValidateInputTypeAndPlacement( - tensorflow::Device* host_device, tensorflow::Device* op_device, TFE_Op* op, - const tensorflow::OpKernel* kernel) { + TFE_Context* ctx, tensorflow::Device* host_device, + tensorflow::Device* op_device, TFE_Op* op, + const tensorflow::OpKernel* kernel, + std::vector* copied_tensors) { const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types(); if (memtypes.size() != op->inputs.size()) { return tensorflow::errors::InvalidArgument( @@ -430,11 +438,42 @@ tensorflow::Status ValidateInputTypeAndPlacement( const tensorflow::Device* actual_device = op->input_devices[i] == nullptr ? host_device : op->input_devices[i]; if (expected_device != actual_device) { - return tensorflow::errors::InvalidArgument( - "cannot compute ", op->name, " as input #", i, - " was expected to be on ", expected_device->name(), - " but is actually on ", actual_device->name(), - " (operation running on ", op_device->name(), ")"); + switch (ctx->policy) { + case TFE_DEVICE_PLACEMENT_EXPLICIT: + return tensorflow::errors::InvalidArgument( + "cannot compute ", op->name, " as input #", i, + " was expected to be on ", expected_device->name(), + " but is actually on ", actual_device->name(), + " (operation running on ", op_device->name(), ")"); + case TFE_DEVICE_PLACEMENT_WARN: + LOG(WARNING) << "before computing " << op->name << " input #" << i + << " was expected to be on " << expected_device->name() + << " but is actually on " << actual_device->name() + << " (operation running on " << op_device->name() + << "). This triggers a copy which can be a performance " + "bottleneck."; + break; + case TFE_DEVICE_PLACEMENT_SILENT: // Do nothing. + break; + } + // We are only here if the policy is warn or silent copies, so we should + // trigger a copy. + TFE_TensorHandle original{op->inputs[i], op->input_devices[i]}; + TF_Status* s = TF_NewStatus(); + TFE_TensorHandle* copied_tensor = TFE_TensorHandleCopyToDevice( + &original, ctx, expected_device->name().c_str(), s); + if (!s->status.ok()) { + tensorflow::Status status = s->status; + delete s; + return tensorflow::errors::Internal( + "Failed copying input tensor from ", actual_device->name(), " to ", + expected_device->name(), " in order to run ", op->name, ": ", + status.error_message()); + } + op->inputs[i] = copied_tensor->t; + copied_tensors->push_back(copied_tensor); + op->input_devices[i] = copied_tensor->d; + delete s; } if (op->inputs[i].dtype() != kernel->input_type(i)) { return tensorflow::errors::InvalidArgument( @@ -477,10 +516,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, } tensorflow::gtl::InsertOrUpdate(&(ctx->kernel_cache), cache_key, kernel); } - status->status = ValidateInputTypeAndPlacement(ctx->devices()[0], device, op, - kernel->kernel()); + std::vector copied_tensors; + status->status = ValidateInputTypeAndPlacement( + ctx, ctx->devices()[0], device, op, kernel->kernel(), &copied_tensors); output_memory_types = &kernel->kernel()->output_memory_types(); if (!status->status.ok()) { + for (auto* t : copied_tensors) { + TFE_DeleteTensorHandle(t); + } return; } // WARNING: kernel->Run utilizes the FunctionLibraryRuntime @@ -492,6 +535,9 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, // sense for FunctionLibraryRuntime to ensure thread-safe access to // FunctionLibraryDefinition?). status->status = kernel->Run(&op->inputs, &outputs); + for (auto* t : copied_tensors) { + TFE_DeleteTensorHandle(t); + } if (!status->status.ok()) return; *num_retvals = std::min(*num_retvals, outputs.size()); for (int i = 0; i < *num_retvals; ++i) { diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h index 201cb222c9..865580c5f3 100644 --- a/tensorflow/c/eager/c_api.h +++ b/tensorflow/c/eager/c_api.h @@ -56,6 +56,22 @@ TF_CAPI_EXPORT extern void TFE_ContextOptionsSetConfig( TFE_ContextOptions* options, const void* proto, size_t proto_len, TF_Status* status); +// Controls how to act when we try to run an operation on a given device but +// some input tensors are not on that device. +typedef enum TFE_ContextDevicePlacementPolicy { + // The default: running operations with input tensors on the wrong device will + // fail. + TFE_DEVICE_PLACEMENT_EXPLICIT = 0, + // Copy the tensor to the right device but log a warning. + TFE_DEVICE_PLACEMENT_WARN = 1, + // Silently copy the tensor, which has a performance cost since the + // operation will be blocked till the copy completes. + TFE_DEVICE_PLACEMENT_SILENT = 2, +} TFE_ContextDevicePlacementPolicy; + +TF_CAPI_EXPORT extern void TFE_ContextOptionsSetDevicePlacementPolicy( + TFE_ContextOptions*, TFE_ContextDevicePlacementPolicy); + // Destroy an options object. TF_CAPI_EXPORT extern void TFE_DeleteContextOptions(TFE_ContextOptions*); diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h index 7a440a5a7e..0971e2ab2f 100644 --- a/tensorflow/c/eager/c_api_internal.h +++ b/tensorflow/c/eager/c_api_internal.h @@ -37,11 +37,14 @@ limitations under the License. struct TFE_ContextOptions { TF_SessionOptions session_options; + TFE_ContextDevicePlacementPolicy policy{TFE_DEVICE_PLACEMENT_EXPLICIT}; }; struct TFE_Context { explicit TFE_Context(TF_Session* s) : session(s) {} + TFE_ContextDevicePlacementPolicy policy; + // TFE_Context is an extension of TF_Session. And TF_Session needs a TF_Graph. TF_Session* session; tensorflow::Rendezvous* rendezvous; diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc index 5344956ee7..4af91b8853 100644 --- a/tensorflow/c/eager/c_api_test.cc +++ b/tensorflow/c/eager/c_api_test.cc @@ -216,6 +216,52 @@ TEST(CAPI, TensorHandleCopyBetweenDevices) { EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); } +TEST(CAPI, TensorHandleSilentCopy) { + std::unique_ptr status( + TF_NewStatus(), TF_DeleteStatus); + TFE_ContextOptions* opts = TFE_NewContextOptions(); + TFE_ContextOptionsSetDevicePlacementPolicy(opts, TFE_DEVICE_PLACEMENT_SILENT); + TFE_Context* ctx = TFE_NewContext(opts, status.get()); + TFE_DeleteContextOptions(opts); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + + TFE_TensorHandle* hcpu = TestMatrixTensorHandle(); + TF_Tensor* t = TFE_TensorHandleResolve(hcpu, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + + TF_DeviceList* devices = TFE_ContextListDevices(ctx, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + const int num_devices = TF_DeviceListCount(devices); + + // Disable the test if no GPU is present. + if (num_devices > 1) { + const int device_to_use = 1; + const string name(TF_DeviceListName(devices, device_to_use, status.get())); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + + TFE_TensorHandle* hgpu = + TFE_TensorHandleCopyToDevice(hcpu, ctx, name.c_str(), status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + + TFE_Op* matmul = MatMulOp(ctx, hcpu, hgpu); + TFE_OpSetDevice(matmul, name.c_str(), status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + TFE_TensorHandle* retvals[1]; + int num_retvals = 1; + TFE_Execute(matmul, &retvals[0], &num_retvals, status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + TFE_DeleteOp(matmul); + TFE_DeleteTensorHandle(retvals[0]); + TFE_DeleteTensorHandle(hgpu); + } + + TF_DeleteDeviceList(devices); + TF_DeleteTensor(t); + TFE_DeleteTensorHandle(hcpu); + TFE_DeleteContext(ctx, status.get()); + EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); +} + TEST(CAPI, Execute) { TF_Status* status = TF_NewStatus(); TFE_ContextOptions* opts = TFE_NewContextOptions(); -- GitLab From e65fbbc9dc608d97977b17e05250b015d65aa027 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 13:03:50 -0700 Subject: [PATCH 214/573] Expose tf.contrib.framework.current_arg_scope() PiperOrigin-RevId: 172922818 --- tensorflow/contrib/framework/__init__.py | 1 + tensorflow/contrib/framework/python/ops/arg_scope.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py index 2081a11f47..8421ba7c04 100644 --- a/tensorflow/contrib/framework/__init__.py +++ b/tensorflow/contrib/framework/__init__.py @@ -37,6 +37,7 @@ See the @{$python/contrib.framework} guide. @@arg_scope @@add_arg_scope +@@current_arg_scope @@has_arg_scope @@arg_scoped_arguments diff --git a/tensorflow/contrib/framework/python/ops/arg_scope.py b/tensorflow/contrib/framework/python/ops/arg_scope.py index 9c194ec202..2bce00fde2 100644 --- a/tensorflow/contrib/framework/python/ops/arg_scope.py +++ b/tensorflow/contrib/framework/python/ops/arg_scope.py @@ -67,6 +67,7 @@ from tensorflow.python.util import tf_decorator __all__ = ['arg_scope', 'add_arg_scope', + 'current_arg_scope', 'has_arg_scope', 'arg_scoped_arguments'] @@ -83,7 +84,7 @@ def _get_arg_stack(): return _ARGSTACK -def _current_arg_scope(): +def current_arg_scope(): stack = _get_arg_stack() return stack[-1] @@ -144,7 +145,7 @@ def arg_scope(list_ops_or_scope, **kwargs): raise TypeError('list_ops_or_scope must either be a list/tuple or reused' 'scope (i.e. dict)') try: - current_scope = _current_arg_scope().copy() + current_scope = current_arg_scope().copy() for op in list_ops_or_scope: key_op = _key_op(op) if not has_arg_scope(op): @@ -172,7 +173,7 @@ def add_arg_scope(func): A tuple with the decorated function func_with_args(). """ def func_with_args(*args, **kwargs): - current_scope = _current_arg_scope() + current_scope = current_arg_scope() current_args = kwargs key_func = _key_op(func) if key_func in current_scope: -- GitLab From d2d9a6c7cc3b4f8c068054082a0fa2f2b95bb3d6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 13:19:42 -0700 Subject: [PATCH 215/573] Add AdaptiveSharedBatchScheduler which processes batches at a variable rate which can be adjusted based on external feedback. For reasonable feedback, this scheduler should deliver better latency than the SharedBatchScheduler. PiperOrigin-RevId: 172924803 --- tensorflow/contrib/batching/BUILD | 22 + .../adaptive_shared_batch_scheduler.h | 463 ++++++++++++++++++ .../adaptive_shared_batch_scheduler_test.cc | 438 +++++++++++++++++ tensorflow/contrib/batching/batch_scheduler.h | 2 +- 4 files changed, 924 insertions(+), 1 deletion(-) create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD index 1555a3427f..ae3f48f1b2 100644 --- a/tensorflow/contrib/batching/BUILD +++ b/tensorflow/contrib/batching/BUILD @@ -69,6 +69,28 @@ tf_cc_test( ], ) +cc_library( + name = "adaptive_shared_batch_scheduler", + hdrs = ["adaptive_shared_batch_scheduler.h"], + deps = [ + ":batch_scheduler", + "//tensorflow/contrib/batching/util:periodic_function_dynamic", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "adaptive_shared_batch_scheduler_test", + srcs = ["adaptive_shared_batch_scheduler_test.cc"], + deps = [ + ":adaptive_shared_batch_scheduler", + "//tensorflow/contrib/batching/test_util:fake_clock_env", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + cc_library( name = "basic_batch_scheduler", hdrs = ["basic_batch_scheduler.h"], diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h new file mode 100644 index 0000000000..ac32f09639 --- /dev/null +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h @@ -0,0 +1,463 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ +#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ + +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/batching/batch_scheduler.h" +#include "tensorflow/contrib/batching/util/periodic_function.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace serving { +namespace internal { +template +class ASBSBatch; + +template +class ASBSQueue; +} // namespace internal + +// Shared batch scheduler designed to minimize latency. The scheduler keeps +// track of a number of queues (one per model or model version) which are +// continuously enqueuing requests. The scheduler groups the requests into +// batches which it periodically sends off for processing (see +// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler +// prioritizes batches by age (i.e. the batch's oldest request) irrespective of +// queue. The scheduler will process the oldest batch at an adjustable rate, +// regardless of batch size. The user can provide feedback to help set this rate +// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc). +// +// The rate (or rather, the corresponding period) is adjusted each time a batch +// is processed, using an exponentially weighted moving average to smooth +// potentially noisy feedback: +// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N +// period *= (1 + K * emwa_feedback) +// +// Some potential use cases: +// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing +// involves serial processing by a device, from a latency perspective it is +// desirable to keep the device evenly loaded, avoiding the need to wait for +// the device to process prior batches. +// feedback = num_pending_on_device() - desired_pending. +// CPU utilization - If the batch processing is cpu dominated, you can reap +// latency gains when underutilized by increasing the processing rate, but +// back the rate off when the load increases to avoid overload. +// feedback = cpu_rate() - desired_cpu_rate. + +template +class AdaptiveSharedBatchScheduler + : public std::enable_shared_from_this< + AdaptiveSharedBatchScheduler> { + public: + struct Options { + // The name to use for the pool of batch threads. + string thread_pool_name = {"batch_threads"}; + // Number of batch processing threads; equivalently the maximum number of + // concurrently running batches. + int64 num_batch_threads = port::NumSchedulableCPUs(); + // The environment to use (typically only overridden by test code). + Env* env = Env::Default(); + // Initial batch scheduling period in microseconds. Will be altered for + // non-zero rate_feedback. + double initial_scheduling_period_micros = 500; + // Minimum batch scheduling period in microseconds. Recommend setting this + // value greater than 0, otherwise it may take a while to recover from a + // sustained time of negative scheduling_period_feedback (which may occur + // under low load). + double min_scheduling_period_micros = 100; + // Maximum batch scheduling period in microseconds. + double max_scheduling_period_micros = 10000; + // Feedback function used to modify the scheduling period each time a batch + // is scheduled. Should return values roughly O(1), with positive values + // resulting in an increased period. + std::function scheduling_period_feedback = [] { return 0.; }; + // To handle potentially noisy scheduling_period_feedback, the period is + // adjusted using an exponentially weighted moving average over the previous + // feedback_smoothing_batches batches. Must be greater than 0. + int64 feedback_smoothing_batches = 10; + }; + + // Ownership is shared between the caller of Create() and any queues created + // via AddQueue(). + static Status Create( + const Options& options, + std::shared_ptr>* scheduler); + + struct QueueOptions { + // Maximum size of each batch. + int max_batch_size = 1000; + // Maximum number of enqueued (i.e. non-scheduled) batches. + int max_enqueued_batches = 10; + }; + + using BatchProcessor = std::function>)>; + + // Adds queue (and its callback) to be managed by this scheduler. + Status AddQueue(const QueueOptions& options, + BatchProcessor process_batch_callback, + std::unique_ptr>* queue); + + private: + // access to AddBatch, RemoveQueue, GetEnv. + friend class internal::ASBSQueue; + + explicit AdaptiveSharedBatchScheduler(const Options& options); + + // Batch scheduling function which runs every scheduling_period_ microseconds. + void ProcessOneBatch(); + + // Notifies scheduler of non-empty batch which is eligible for processing. + void AddBatch(internal::ASBSBatch*); + + // Removes queue from scheduler. + void RemoveQueue(const internal::ASBSQueue* queue); + + Env* GetEnv() const { return options_.env; } + + const Options options_; + + struct BatchCompare { + bool operator()(const internal::ASBSBatch* a, + const internal::ASBSBatch* b); + }; + + // Collection of batches added by AddBatch, ordered by age. Owned by scheduler + // until they are released for processing. + std::priority_queue*, + std::vector*>, BatchCompare> + batches_ GUARDED_BY(mu_); + + // Unowned queues and callbacks added by AddQueue. + std::unordered_map*, BatchProcessor> + queues_and_callbacks_ GUARDED_BY(mu_); + + mutex mu_; + + // Responsible for running ProcessOneBatch. PeriodicFunction was used in order + // to check for deletion so that the thread can be shut down. + std::unique_ptr scheduling_thread_; + + // Responsible for running the batch processing callbacks. + std::unique_ptr batch_thread_pool_; + + // Time interval in microseconds between successive ProcessOneBatch calls. + double scheduling_period_; + + // Exponentially weighted moving average of + // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch + // call. + double ewma_feedback_ = 0; + + TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler); +}; + +////////////////////////////////////////////////////////// +// Implementation details follow. API users need not read. + +namespace internal { +// Consolidates tasks into batches, passing them off to the +// AdaptiveSharedBatchScheduler for processing. +template +class ASBSQueue : public BatchScheduler { + public: + using QueueOptions = + typename AdaptiveSharedBatchScheduler::QueueOptions; + + ASBSQueue(std::shared_ptr> scheduler, + const QueueOptions& options); + + ~ASBSQueue() override; + + // Adds task to current batch. Fails if the task size is larger than the batch + // size or if the current batch is full and this queue's number of outstanding + // batches is at its maximum. + Status Schedule(std::unique_ptr* task) override; + + // Number of tasks waiting to be scheduled. + size_t NumEnqueuedTasks() const override; + + // Number of size 1 tasks which could currently be scheduled without failing. + size_t SchedulingCapacity() const override; + + // Notifies queue that a batch is about to be scheduled; the queue should not + // place any more tasks in this batch. + void ReleaseBatch(const ASBSBatch* batch); + + private: + std::shared_ptr> scheduler_; + const QueueOptions options_; + // Owned by scheduler_. + ASBSBatch* current_batch_ GUARDED_BY(mu_) = nullptr; + int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0; + int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0; + mutable mutex mu_; + TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue); +}; + +// Batch which remembers when and by whom it was created. +template +class ASBSBatch : public Batch { + public: + ASBSBatch(ASBSQueue* queue, int64 creation_time_micros) + : queue_(queue), creation_time_micros_(creation_time_micros) {} + + ~ASBSBatch() override {} + + ASBSQueue* queue() const { return queue_; } + + int64 creation_time_micros() const { return creation_time_micros_; } + + private: + ASBSQueue* queue_; + const int64 creation_time_micros_; + TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch); +}; +} // namespace internal + +// ---------------- AdaptiveSharedBatchScheduler ---------------- + +template +Status AdaptiveSharedBatchScheduler::Create( + const Options& options, + std::shared_ptr>* scheduler) { + if (options.num_batch_threads < 1) { + return errors::InvalidArgument("num_batch_threads must be positive; was ", + options.num_batch_threads); + } + if (options.min_scheduling_period_micros < 0) { + return errors::InvalidArgument( + "min_scheduling_period_micros must be >= 0; was ", + options.min_scheduling_period_micros); + } + if (options.min_scheduling_period_micros > + options.initial_scheduling_period_micros) { + return errors::InvalidArgument( + "initial_scheduling_period_micros (", + options.initial_scheduling_period_micros, + ") must be >= min_scheduling_period_micros (", + options.min_scheduling_period_micros, ")"); + } + if (options.initial_scheduling_period_micros > + options.max_scheduling_period_micros) { + return errors::InvalidArgument( + "initial_scheduling_period_micros (", + options.initial_scheduling_period_micros, + ") must be <= max_scheduling_period_micros (", + options.max_scheduling_period_micros, ")"); + } + if (options.feedback_smoothing_batches < 1) { + return errors::InvalidArgument( + "feedback_smoothing_batches must be positive; was ", + options.feedback_smoothing_batches); + } + scheduler->reset(new AdaptiveSharedBatchScheduler(options)); + return Status::OK(); +} + +template +AdaptiveSharedBatchScheduler::AdaptiveSharedBatchScheduler( + const Options& options) + : options_(options), + scheduling_period_(options.initial_scheduling_period_micros) { + PeriodicFunction::Options opts; + opts.thread_name_prefix = "scheduling_thread"; + opts.env = GetEnv(); + scheduling_thread_.reset( + new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts)); + batch_thread_pool_.reset(new thread::ThreadPool( + GetEnv(), options.thread_pool_name, options.num_batch_threads)); +} + +template +Status AdaptiveSharedBatchScheduler::AddQueue( + const QueueOptions& options, BatchProcessor process_batch_callback, + std::unique_ptr>* queue) { + if (options.max_batch_size <= 0) { + return errors::InvalidArgument("max_batch_size must be positive; was ", + options.max_batch_size); + } + if (options.max_enqueued_batches <= 0) { + return errors::InvalidArgument( + "max_enqueued_batches must be positive; was ", + options.max_enqueued_batches); + } + internal::ASBSQueue* asbs_queue_raw; + queue->reset(asbs_queue_raw = new internal::ASBSQueue( + this->shared_from_this(), options)); + mutex_lock l(mu_); + queues_and_callbacks_[asbs_queue_raw] = process_batch_callback; + return Status::OK(); +} + +template +void AdaptiveSharedBatchScheduler::AddBatch( + internal::ASBSBatch* batch) { + mutex_lock l(mu_); + batches_.push(batch); +} + +template +void AdaptiveSharedBatchScheduler::RemoveQueue( + const internal::ASBSQueue* queue) { + mutex_lock l(mu_); + queues_and_callbacks_.erase(queue); +} + +template +void AdaptiveSharedBatchScheduler::ProcessOneBatch() { + static const double kFeedbackMultiplier = .001; + internal::ASBSBatch* batch = nullptr; + BatchProcessor callback; + const int64 start_time_micros = GetEnv()->NowMicros(); + { + mutex_lock l(mu_); + if (!batches_.empty()) { + batch = batches_.top(); + batches_.pop(); + callback = queues_and_callbacks_[batch->queue()]; + } + } + if (batch != nullptr) { + double feedback = options_.scheduling_period_feedback(); + const int64 N = options_.feedback_smoothing_batches; + ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N; + scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_); + if (scheduling_period_ < options_.min_scheduling_period_micros) { + scheduling_period_ = options_.min_scheduling_period_micros; + } else if (scheduling_period_ > options_.max_scheduling_period_micros) { + scheduling_period_ = options_.max_scheduling_period_micros; + } + // Queue may destroy itself after ReleaseBatch is called. + batch->queue()->ReleaseBatch(batch); + batch_thread_pool_->Schedule([callback, batch] { + callback(std::unique_ptr>(batch)); + }); + } + const int64 sleep_time = + scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros); + if (sleep_time > 0) { + GetEnv()->SleepForMicroseconds(sleep_time); + } +} + +template +bool AdaptiveSharedBatchScheduler::BatchCompare::operator()( + const internal::ASBSBatch* a, + const internal::ASBSBatch* b) { + return a->creation_time_micros() > b->creation_time_micros(); +} + +// ---------------- ASBSQueue ---------------- + +namespace internal { +template +ASBSQueue::ASBSQueue( + std::shared_ptr> scheduler, + const QueueOptions& options) + : scheduler_(scheduler), options_(options) {} + +template +ASBSQueue::~ASBSQueue() { + // Wait until last batch has been scheduled. + const int kSleepMicros = 1000; + for (;;) { + { + mutex_lock l(mu_); + if (num_enqueued_batches_ == 0) { + break; + } + } + scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros); + } + scheduler_->RemoveQueue(this); +} + +template +Status ASBSQueue::Schedule(std::unique_ptr* task) { + bool added_new_batch = false; + size_t size = (*task)->size(); + if (size > options_.max_batch_size) { + return errors::InvalidArgument("Task size ", size, + " is larger than maximum batch size ", + options_.max_batch_size); + } + { + mutex_lock l(mu_); + // Current batch is full, create another if allowed. + if (current_batch_ && + current_batch_->size() + size > options_.max_batch_size) { + if (num_enqueued_batches_ >= options_.max_enqueued_batches) { + return errors::Unavailable("The batch scheduling queue is full"); + } + current_batch_->Close(); + current_batch_ = nullptr; + } + if (!current_batch_) { + added_new_batch = true; + num_enqueued_batches_++; + current_batch_ = + new ASBSBatch(this, scheduler_->GetEnv()->NowMicros()); + } + current_batch_->AddTask(std::move(*task)); + num_enqueued_tasks_++; + } + if (added_new_batch) scheduler_->AddBatch(current_batch_); + return Status::OK(); +} + +template +void ASBSQueue::ReleaseBatch(const ASBSBatch* batch) { + mutex_lock l(mu_); + num_enqueued_batches_--; + num_enqueued_tasks_ -= batch->num_tasks(); + if (batch == current_batch_) { + current_batch_->Close(); + current_batch_ = nullptr; + } +} + +template +size_t ASBSQueue::NumEnqueuedTasks() const { + mutex_lock l(mu_); + return num_enqueued_tasks_; +} + +template +size_t ASBSQueue::SchedulingCapacity() const { + mutex_lock l(mu_); + const int current_batch_capacity = + current_batch_ ? options_.max_batch_size - current_batch_->size() : 0; + const int spare_batches = + options_.max_enqueued_batches - num_enqueued_batches_; + return spare_batches * options_.max_batch_size + current_batch_capacity; +} +} // namespace internal +} // namespace serving +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc new file mode 100644 index 0000000000..a07cd6d834 --- /dev/null +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc @@ -0,0 +1,438 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h" + +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace serving { +namespace anonymous { + +class FakeTask : public BatchTask { + public: + explicit FakeTask(size_t size) : size_(size) {} + + ~FakeTask() override = default; + + size_t size() const override { return size_; } + + private: + const size_t size_; + + TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); +}; + +// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on +// that task. Returns the resulting status. +Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { + std::unique_ptr task(new FakeTask(task_size)); + Status status = scheduler->Schedule(&task); + // Schedule() should have consumed 'task' iff it returned Status::OK. + CHECK_EQ(status.ok(), task == nullptr); + return status; +} + +// Creates a thread that waits on 'start' and then advances the fake clock in +// 'env' in a loop until 'stop' is notified. Useful for allowing objects that +// use the clock to be destroyed. +std::unique_ptr CreateFakeClockAdvancerThread( + test_util::FakeClockEnv* env, Notification* start, Notification* stop) { + return std::unique_ptr(Env::Default()->StartThread( + {}, "FakeClockAdvancerThread", [env, start, stop] { + start->WaitForNotification(); + while (!stop->HasBeenNotified()) { + env->AdvanceByMicroseconds(10); + Env::Default()->SleepForMicroseconds(10); + } + })); +} + +TEST(AdaptiveSharedBatchSchedulerTest, Basic) { + for (const bool delete_scheduler_early : {false, true}) { + for (const bool delete_queue_1_early : {false, true}) { + int queue_0_tasks = 0; + auto queue_0_callback = + [&queue_0_tasks](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_0_tasks += batch->task(i).size(); + } + }; + int queue_1_tasks = 0; + auto queue_1_callback = + [&queue_1_tasks](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_1_tasks += batch->task(i).size(); + } + }; + { + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create({}, &scheduler)); + + // Create two queues. + std::unique_ptr> queue_0; + TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0)); + std::unique_ptr> queue_1; + TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1)); + + if (delete_scheduler_early) { + // Delete our copy of the scheduler. The queues should keep it alive + // under the covers. + scheduler = nullptr; + } + // Submit tasks to the two queues, and (optionally) remove the queues. + TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); + TF_ASSERT_OK(ScheduleTask(3, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(4, queue_1.get())); + if (delete_queue_1_early) { + queue_1 = nullptr; + } + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + } + EXPECT_EQ(queue_0_tasks, 9); + EXPECT_EQ(queue_1_tasks, 6); + } + } +} + +TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) { + using Scheduler = AdaptiveSharedBatchScheduler; + std::shared_ptr scheduler; + Scheduler::Options options; + options.num_batch_threads = 0; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 50; + options.max_scheduling_period_micros = 100; + options.initial_scheduling_period_micros = 1; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 50; + options.max_scheduling_period_micros = 100; + options.initial_scheduling_period_micros = 1000; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 100; + options.max_scheduling_period_micros = 50; + options.initial_scheduling_period_micros = 75; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.feedback_smoothing_batches = 0; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); +} + +TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue_0; + std::unique_ptr> queue_1; + int queue_0_tasks = 0; + int queue_1_tasks = 0; + auto queue_0_callback = [&queue_0_tasks, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_0_tasks += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + auto queue_1_callback = [&queue_1_tasks, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_1_tasks += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + AdaptiveSharedBatchScheduler::QueueOptions queue_options; + queue_options.max_batch_size = 10; + queue_options.max_enqueued_batches = 0; + // Queue must have max_enqueued_batchs > 1. + EXPECT_FALSE( + scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok()); + queue_options.max_enqueued_batches = 2; + TF_ASSERT_OK( + scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); + queue_options.max_batch_size = 0; + // Queue must have max_batch_size > 0. + EXPECT_FALSE( + scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok()); + queue_options.max_batch_size = 2; + queue_options.max_enqueued_batches = 1; + TF_ASSERT_OK( + scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Task larger than max_batch_size shouldn't schedule. + EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok()); + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + env.AdvanceByMicroseconds(1); + + // Task larger than max_batch_size shouldn't schedule. + EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok()); + TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); + TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); + env.AdvanceByMicroseconds(1); + // Exceeds max_enqueued_batches, shouldn't schedule. + EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok()); + + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + // Exceeds max_enqueued_batches, shouldn't schedule. + EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok()); + TF_ASSERT_OK(ScheduleTask(4, queue_0.get())); + + // Batches should be processed in order from oldest to newest. + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 10); + EXPECT_EQ(queue_1_tasks, 0); + + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 10); + EXPECT_EQ(queue_1_tasks, 2); + + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 19); + EXPECT_EQ(queue_1_tasks, 2); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + double feedback = 0; + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.min_scheduling_period_micros = 200; + options.max_scheduling_period_micros = 2000; + options.env = &env; + options.scheduling_period_feedback = [&feedback] { return feedback; }; + options.feedback_smoothing_batches = 1; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + + TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 6 batches. + for (int i = 0; i < 6; i++) { + TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); + env.AdvanceByMicroseconds(1); + } + feedback = -500; + env.AdvanceByMicroseconds(994); + env.BlockUntilThreadsAsleep(2); // scheduling period = 500 usec. + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(500); + env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. + EXPECT_EQ(scheduled_items, 901); + feedback = 0; + env.AdvanceByMicroseconds(250); + env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. + EXPECT_EQ(scheduled_items, 902); + feedback = 10000; // large feedback should hit max_scheduling_period. + env.AdvanceByMicroseconds(250); + env.BlockUntilThreadsAsleep(2); // scheduling period = 2000 usec. + EXPECT_EQ(scheduled_items, 903); + feedback = -10000; // large feedback should hit min_scheduling_period. + env.AdvanceByMicroseconds(1999); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 903); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); // scheduling period = 200 usec. + EXPECT_EQ(scheduled_items, 904); + env.AdvanceByMicroseconds(200); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 905); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + double feedback = 0; + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + options.scheduling_period_feedback = [&feedback] { return feedback; }; + options.feedback_smoothing_batches = 3; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + + TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 4 batches. + for (int i = 0; i < 4; i++) { + TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); + env.AdvanceByMicroseconds(1); + } + feedback = -300; + env.AdvanceByMicroseconds(996); + env.BlockUntilThreadsAsleep(2); + // ewma_feedback = 100, scheduling_period = 900. + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(899); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + // ewma_feedback = 167, scheduling_period = 750. + EXPECT_EQ(scheduled_items, 901); + env.AdvanceByMicroseconds(749); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 901); + feedback = 1000 / 3.; + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + // emwa_feedback = 0, scheduling_period = 750. + EXPECT_EQ(scheduled_items, 902); + env.AdvanceByMicroseconds(749); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 902); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 903); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + AdaptiveSharedBatchScheduler::QueueOptions queue_options; + queue_options.max_batch_size = 10; + queue_options.max_enqueued_batches = 10; + TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 3 tasks. + EXPECT_EQ(queue->NumEnqueuedTasks(), 0); + EXPECT_EQ(queue->SchedulingCapacity(), 100); + TF_ASSERT_OK(ScheduleTask(5, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 1); + EXPECT_EQ(queue->SchedulingCapacity(), 95); + env.AdvanceByMicroseconds(1); + TF_ASSERT_OK(ScheduleTask(6, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 2); + EXPECT_EQ(queue->SchedulingCapacity(), 84); + env.AdvanceByMicroseconds(1); + TF_ASSERT_OK(ScheduleTask(1, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 3); + EXPECT_EQ(queue->SchedulingCapacity(), 83); + + env.AdvanceByMicroseconds(998); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 5); + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 7); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} +} // namespace anonymous +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h index 7c41ad8818..a5072f439a 100644 --- a/tensorflow/contrib/batching/batch_scheduler.h +++ b/tensorflow/contrib/batching/batch_scheduler.h @@ -78,7 +78,7 @@ template class Batch { public: Batch() = default; - ~Batch(); // Blocks until the batch is closed. + virtual ~Batch(); // Blocks until the batch is closed. // Appends 'task' to the batch. After calling AddTask(), the newly-added task // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1). -- GitLab From 32eb07bf7b4cf5c9f5ee14e1f4cbe18b1eba6c4d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 13:44:10 -0700 Subject: [PATCH 216/573] Simplify the graph generated for contrib/summaries in the "always summarize" and "never summarize" cases by skipping the `cond`. PiperOrigin-RevId: 172928083 --- tensorflow/contrib/summary/BUILD | 2 +- tensorflow/contrib/summary/summary_ops.py | 11 +++++------ tensorflow/contrib/summary/summary_ops_test.py | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD index d09ad48e10..bcb2d74b4a 100644 --- a/tensorflow/contrib/summary/BUILD +++ b/tensorflow/contrib/summary/BUILD @@ -43,9 +43,9 @@ py_library( deps = [ ":gen_summary_ops", "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:layers_base", "//tensorflow/python:summary_op_util", "//tensorflow/python:training", "//tensorflow/python/eager:context", diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index c8d0c14e19..ba3619bfc9 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -24,11 +24,10 @@ from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.ops import control_flow_ops +from tensorflow.python.layers import utils from tensorflow.python.ops import summary_op_util from tensorflow.python.training import training_util - # Name for a collection which is expected to have at most a single boolean # Tensor. If this tensor is True the summary ops will record summaries. _SHOULD_RECORD_SUMMARIES_NAME = "ShouldRecordSummaries" @@ -38,7 +37,7 @@ def should_record_summaries(): """Returns boolean Tensor which is true if summaries should be recorded.""" should_record_collection = ops.get_collection(_SHOULD_RECORD_SUMMARIES_NAME) if not should_record_collection: - return constant_op.constant(False) + return False if len(should_record_collection) != 1: raise ValueError( "More than one tensor specified for whether summaries " @@ -56,13 +55,13 @@ def record_summaries_every_n_global_steps(n): def always_record_summaries(): """Sets the should_record_summaries Tensor to always true.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) - collection_ref[:] = [constant_op.constant(True)] + collection_ref[:] = [True] def never_record_summaries(): """Sets the should_record_summaries Tensor to always false.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) - collection_ref[:] = [constant_op.constant(False)] + collection_ref[:] = [False] def create_summary_file_writer(logdir, @@ -106,7 +105,7 @@ def summary_writer_function(name, tensor, function, family=None): function(tag, scope) return True - return control_flow_ops.cond( + return utils.smart_cond( should_record_summaries(), record, _nothing, name="") diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index 6958ee8dd8..2cd4fce5b3 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -40,9 +40,9 @@ class TargetTest(test_util.TensorFlowTestCase): summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t0') def testShouldRecordSummary(self): - self.assertFalse(summary_ops.should_record_summaries().numpy()) + self.assertFalse(summary_ops.should_record_summaries()) summary_ops.always_record_summaries() - self.assertTrue(summary_ops.should_record_summaries().numpy()) + self.assertTrue(summary_ops.should_record_summaries()) def testSummaryOps(self): training_util.get_or_create_global_step() -- GitLab From afd9224b3f6148e4c115e0da8ad4e57a8b47e383 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 13:59:55 -0700 Subject: [PATCH 217/573] Updating to latest stable version of llvm repo (same used to generate latest in gcr.io/cloud-marketplace/google/clang-debian8) PiperOrigin-RevId: 172930105 --- tensorflow/tools/ci_build/install/build_and_install_clang.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/build_and_install_clang.sh b/tensorflow/tools/ci_build/install/build_and_install_clang.sh index 3fb9964948..9966434477 100755 --- a/tensorflow/tools/ci_build/install/build_and_install_clang.sh +++ b/tensorflow/tools/ci_build/install/build_and_install_clang.sh @@ -16,7 +16,7 @@ set -ex -LLVM_SVN_REVISION="299268" +LLVM_SVN_REVISION="314281" CLANG_TMP_DIR=/tmp/clang-build mkdir "$CLANG_TMP_DIR" -- GitLab From b2dcbca94c928181986488f3dccdcc9568926988 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Fri, 20 Oct 2017 14:23:37 -0700 Subject: [PATCH 218/573] Add a BUILD file for makefile build. PiperOrigin-RevId: 172933558 --- tensorflow/BUILD | 2 ++ tensorflow/contrib/makefile/BUILD | 31 +++++++++++++++++++ tensorflow/contrib/makefile/Makefile | 3 +- .../contrib/makefile/build_all_linux.sh | 3 +- tensorflow/python/kernel_tests/BUILD | 5 +-- tensorflow/tools/benchmark/BUILD | 9 ++++++ third_party/eigen3/BUILD | 9 ++++++ third_party/fft2d/BUILD | 9 ++++++ 8 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 tensorflow/contrib/makefile/BUILD diff --git a/tensorflow/BUILD b/tensorflow/BUILD index d7d6d5fc77..d4396bacbf 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -400,6 +400,7 @@ filegroup( "//tensorflow/contrib/linear_optimizer:all_files", "//tensorflow/contrib/lookup:all_files", "//tensorflow/contrib/losses:all_files", + "//tensorflow/contrib/makefile:all_files", "//tensorflow/contrib/meta_graph_transform:all_files", "//tensorflow/contrib/metrics:all_files", "//tensorflow/contrib/mpi_collectives:all_files", @@ -513,6 +514,7 @@ filegroup( "//tensorflow/tools/api/golden:all_files", "//tensorflow/tools/api/lib:all_files", "//tensorflow/tools/api/tests:all_files", + "//tensorflow/tools/benchmark:all_files", "//tensorflow/tools/build_info:all_files", "//tensorflow/tools/common:all_files", "//tensorflow/tools/compatibility:all_files", diff --git a/tensorflow/contrib/makefile/BUILD b/tensorflow/contrib/makefile/BUILD new file mode 100644 index 0000000000..a8dd59f32a --- /dev/null +++ b/tensorflow/contrib/makefile/BUILD @@ -0,0 +1,31 @@ +# Necessary build rules for makefile build in our CI. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:private"]) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) + +sh_test( + name = "build_all_linux", + size = "enormous", + srcs = ["build_all_linux.sh"], + data = [ + "//tensorflow:all_opensource_files", + "//third_party/eigen3:all_files", + "//third_party/fft2d:all_files", + ], + tags = [ + "manual", + "no_gpu", + "no_oss", + "notap", + ], +) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 3dcff3d4a3..cb23dd6dab 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -14,7 +14,7 @@ # Host compilation settings # Find where we're running from, so we can store generated files here. -MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +MAKEFILE_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) HAS_GEN_HOST_PROTOC := \ $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\ echo "false") @@ -71,6 +71,7 @@ HOST_LDOPTS += -L/usr/local/lib HOST_INCLUDES := \ -I. \ +-I$(MAKEFILE_DIR)/../../../ \ -I$(MAKEFILE_DIR)/downloads/ \ -I$(MAKEFILE_DIR)/downloads/eigen \ -I$(MAKEFILE_DIR)/downloads/gemmlowp \ diff --git a/tensorflow/contrib/makefile/build_all_linux.sh b/tensorflow/contrib/makefile/build_all_linux.sh index 5d73f697f4..a440633cfc 100755 --- a/tensorflow/contrib/makefile/build_all_linux.sh +++ b/tensorflow/contrib/makefile/build_all_linux.sh @@ -44,4 +44,5 @@ tensorflow/contrib/makefile/compile_linux_protobuf.sh # Build TensorFlow. make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ OPTFLAGS="-O3 -march=native" \ - HOST_CXXFLAGS="--std=c++11 -march=native" + HOST_CXXFLAGS="--std=c++11 -march=native" \ + MAKEFILE_DIR=$SCRIPT_DIR diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index e6848edc12..0e36c3498a 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2700,10 +2700,7 @@ tf_py_test( "//tensorflow/python:variables", ], shard_count = 3, - tags = [ - "no_windows_gpu", - "nozapfhahn", - ], + tags = ["no_windows_gpu"], ) tf_py_test( diff --git a/tensorflow/tools/benchmark/BUILD b/tensorflow/tools/benchmark/BUILD index 048035f2b1..caa6629c49 100644 --- a/tensorflow/tools/benchmark/BUILD +++ b/tensorflow/tools/benchmark/BUILD @@ -89,3 +89,12 @@ tf_cc_binary( visibility = ["//visibility:public"], deps = [":benchmark_model_lib"], ) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/third_party/eigen3/BUILD b/third_party/eigen3/BUILD index f38a26717e..ad87477b7a 100644 --- a/third_party/eigen3/BUILD +++ b/third_party/eigen3/BUILD @@ -38,3 +38,12 @@ cc_library( "@local_config_sycl//sycl:sycl", ], ) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/third_party/fft2d/BUILD b/third_party/fft2d/BUILD index 93ea06e81b..8135442482 100644 --- a/third_party/fft2d/BUILD +++ b/third_party/fft2d/BUILD @@ -28,3 +28,12 @@ filegroup( name = "fft2d_headers_srcs", srcs = ["fft.h"], ) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) -- GitLab From d1e7382af7b99dad2455d9b7eaf34989a75f26d2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 14:48:27 -0700 Subject: [PATCH 219/573] Automated g4 rollback of changelist 172924803 PiperOrigin-RevId: 172936802 --- tensorflow/contrib/batching/BUILD | 22 - .../adaptive_shared_batch_scheduler.h | 463 ------------------ .../adaptive_shared_batch_scheduler_test.cc | 438 ----------------- tensorflow/contrib/batching/batch_scheduler.h | 2 +- 4 files changed, 1 insertion(+), 924 deletions(-) delete mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h delete mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD index ae3f48f1b2..1555a3427f 100644 --- a/tensorflow/contrib/batching/BUILD +++ b/tensorflow/contrib/batching/BUILD @@ -69,28 +69,6 @@ tf_cc_test( ], ) -cc_library( - name = "adaptive_shared_batch_scheduler", - hdrs = ["adaptive_shared_batch_scheduler.h"], - deps = [ - ":batch_scheduler", - "//tensorflow/contrib/batching/util:periodic_function_dynamic", - "//tensorflow/core:lib", - ], -) - -tf_cc_test( - name = "adaptive_shared_batch_scheduler_test", - srcs = ["adaptive_shared_batch_scheduler_test.cc"], - deps = [ - ":adaptive_shared_batch_scheduler", - "//tensorflow/contrib/batching/test_util:fake_clock_env", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - cc_library( name = "basic_batch_scheduler", hdrs = ["basic_batch_scheduler.h"], diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h deleted file mode 100644 index ac32f09639..0000000000 --- a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h +++ /dev/null @@ -1,463 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ -#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ - -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/batching/batch_scheduler.h" -#include "tensorflow/contrib/batching/util/periodic_function.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/platform/cpu_info.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace serving { -namespace internal { -template -class ASBSBatch; - -template -class ASBSQueue; -} // namespace internal - -// Shared batch scheduler designed to minimize latency. The scheduler keeps -// track of a number of queues (one per model or model version) which are -// continuously enqueuing requests. The scheduler groups the requests into -// batches which it periodically sends off for processing (see -// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler -// prioritizes batches by age (i.e. the batch's oldest request) irrespective of -// queue. The scheduler will process the oldest batch at an adjustable rate, -// regardless of batch size. The user can provide feedback to help set this rate -// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc). -// -// The rate (or rather, the corresponding period) is adjusted each time a batch -// is processed, using an exponentially weighted moving average to smooth -// potentially noisy feedback: -// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N -// period *= (1 + K * emwa_feedback) -// -// Some potential use cases: -// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing -// involves serial processing by a device, from a latency perspective it is -// desirable to keep the device evenly loaded, avoiding the need to wait for -// the device to process prior batches. -// feedback = num_pending_on_device() - desired_pending. -// CPU utilization - If the batch processing is cpu dominated, you can reap -// latency gains when underutilized by increasing the processing rate, but -// back the rate off when the load increases to avoid overload. -// feedback = cpu_rate() - desired_cpu_rate. - -template -class AdaptiveSharedBatchScheduler - : public std::enable_shared_from_this< - AdaptiveSharedBatchScheduler> { - public: - struct Options { - // The name to use for the pool of batch threads. - string thread_pool_name = {"batch_threads"}; - // Number of batch processing threads; equivalently the maximum number of - // concurrently running batches. - int64 num_batch_threads = port::NumSchedulableCPUs(); - // The environment to use (typically only overridden by test code). - Env* env = Env::Default(); - // Initial batch scheduling period in microseconds. Will be altered for - // non-zero rate_feedback. - double initial_scheduling_period_micros = 500; - // Minimum batch scheduling period in microseconds. Recommend setting this - // value greater than 0, otherwise it may take a while to recover from a - // sustained time of negative scheduling_period_feedback (which may occur - // under low load). - double min_scheduling_period_micros = 100; - // Maximum batch scheduling period in microseconds. - double max_scheduling_period_micros = 10000; - // Feedback function used to modify the scheduling period each time a batch - // is scheduled. Should return values roughly O(1), with positive values - // resulting in an increased period. - std::function scheduling_period_feedback = [] { return 0.; }; - // To handle potentially noisy scheduling_period_feedback, the period is - // adjusted using an exponentially weighted moving average over the previous - // feedback_smoothing_batches batches. Must be greater than 0. - int64 feedback_smoothing_batches = 10; - }; - - // Ownership is shared between the caller of Create() and any queues created - // via AddQueue(). - static Status Create( - const Options& options, - std::shared_ptr>* scheduler); - - struct QueueOptions { - // Maximum size of each batch. - int max_batch_size = 1000; - // Maximum number of enqueued (i.e. non-scheduled) batches. - int max_enqueued_batches = 10; - }; - - using BatchProcessor = std::function>)>; - - // Adds queue (and its callback) to be managed by this scheduler. - Status AddQueue(const QueueOptions& options, - BatchProcessor process_batch_callback, - std::unique_ptr>* queue); - - private: - // access to AddBatch, RemoveQueue, GetEnv. - friend class internal::ASBSQueue; - - explicit AdaptiveSharedBatchScheduler(const Options& options); - - // Batch scheduling function which runs every scheduling_period_ microseconds. - void ProcessOneBatch(); - - // Notifies scheduler of non-empty batch which is eligible for processing. - void AddBatch(internal::ASBSBatch*); - - // Removes queue from scheduler. - void RemoveQueue(const internal::ASBSQueue* queue); - - Env* GetEnv() const { return options_.env; } - - const Options options_; - - struct BatchCompare { - bool operator()(const internal::ASBSBatch* a, - const internal::ASBSBatch* b); - }; - - // Collection of batches added by AddBatch, ordered by age. Owned by scheduler - // until they are released for processing. - std::priority_queue*, - std::vector*>, BatchCompare> - batches_ GUARDED_BY(mu_); - - // Unowned queues and callbacks added by AddQueue. - std::unordered_map*, BatchProcessor> - queues_and_callbacks_ GUARDED_BY(mu_); - - mutex mu_; - - // Responsible for running ProcessOneBatch. PeriodicFunction was used in order - // to check for deletion so that the thread can be shut down. - std::unique_ptr scheduling_thread_; - - // Responsible for running the batch processing callbacks. - std::unique_ptr batch_thread_pool_; - - // Time interval in microseconds between successive ProcessOneBatch calls. - double scheduling_period_; - - // Exponentially weighted moving average of - // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch - // call. - double ewma_feedback_ = 0; - - TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler); -}; - -////////////////////////////////////////////////////////// -// Implementation details follow. API users need not read. - -namespace internal { -// Consolidates tasks into batches, passing them off to the -// AdaptiveSharedBatchScheduler for processing. -template -class ASBSQueue : public BatchScheduler { - public: - using QueueOptions = - typename AdaptiveSharedBatchScheduler::QueueOptions; - - ASBSQueue(std::shared_ptr> scheduler, - const QueueOptions& options); - - ~ASBSQueue() override; - - // Adds task to current batch. Fails if the task size is larger than the batch - // size or if the current batch is full and this queue's number of outstanding - // batches is at its maximum. - Status Schedule(std::unique_ptr* task) override; - - // Number of tasks waiting to be scheduled. - size_t NumEnqueuedTasks() const override; - - // Number of size 1 tasks which could currently be scheduled without failing. - size_t SchedulingCapacity() const override; - - // Notifies queue that a batch is about to be scheduled; the queue should not - // place any more tasks in this batch. - void ReleaseBatch(const ASBSBatch* batch); - - private: - std::shared_ptr> scheduler_; - const QueueOptions options_; - // Owned by scheduler_. - ASBSBatch* current_batch_ GUARDED_BY(mu_) = nullptr; - int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0; - int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0; - mutable mutex mu_; - TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue); -}; - -// Batch which remembers when and by whom it was created. -template -class ASBSBatch : public Batch { - public: - ASBSBatch(ASBSQueue* queue, int64 creation_time_micros) - : queue_(queue), creation_time_micros_(creation_time_micros) {} - - ~ASBSBatch() override {} - - ASBSQueue* queue() const { return queue_; } - - int64 creation_time_micros() const { return creation_time_micros_; } - - private: - ASBSQueue* queue_; - const int64 creation_time_micros_; - TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch); -}; -} // namespace internal - -// ---------------- AdaptiveSharedBatchScheduler ---------------- - -template -Status AdaptiveSharedBatchScheduler::Create( - const Options& options, - std::shared_ptr>* scheduler) { - if (options.num_batch_threads < 1) { - return errors::InvalidArgument("num_batch_threads must be positive; was ", - options.num_batch_threads); - } - if (options.min_scheduling_period_micros < 0) { - return errors::InvalidArgument( - "min_scheduling_period_micros must be >= 0; was ", - options.min_scheduling_period_micros); - } - if (options.min_scheduling_period_micros > - options.initial_scheduling_period_micros) { - return errors::InvalidArgument( - "initial_scheduling_period_micros (", - options.initial_scheduling_period_micros, - ") must be >= min_scheduling_period_micros (", - options.min_scheduling_period_micros, ")"); - } - if (options.initial_scheduling_period_micros > - options.max_scheduling_period_micros) { - return errors::InvalidArgument( - "initial_scheduling_period_micros (", - options.initial_scheduling_period_micros, - ") must be <= max_scheduling_period_micros (", - options.max_scheduling_period_micros, ")"); - } - if (options.feedback_smoothing_batches < 1) { - return errors::InvalidArgument( - "feedback_smoothing_batches must be positive; was ", - options.feedback_smoothing_batches); - } - scheduler->reset(new AdaptiveSharedBatchScheduler(options)); - return Status::OK(); -} - -template -AdaptiveSharedBatchScheduler::AdaptiveSharedBatchScheduler( - const Options& options) - : options_(options), - scheduling_period_(options.initial_scheduling_period_micros) { - PeriodicFunction::Options opts; - opts.thread_name_prefix = "scheduling_thread"; - opts.env = GetEnv(); - scheduling_thread_.reset( - new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts)); - batch_thread_pool_.reset(new thread::ThreadPool( - GetEnv(), options.thread_pool_name, options.num_batch_threads)); -} - -template -Status AdaptiveSharedBatchScheduler::AddQueue( - const QueueOptions& options, BatchProcessor process_batch_callback, - std::unique_ptr>* queue) { - if (options.max_batch_size <= 0) { - return errors::InvalidArgument("max_batch_size must be positive; was ", - options.max_batch_size); - } - if (options.max_enqueued_batches <= 0) { - return errors::InvalidArgument( - "max_enqueued_batches must be positive; was ", - options.max_enqueued_batches); - } - internal::ASBSQueue* asbs_queue_raw; - queue->reset(asbs_queue_raw = new internal::ASBSQueue( - this->shared_from_this(), options)); - mutex_lock l(mu_); - queues_and_callbacks_[asbs_queue_raw] = process_batch_callback; - return Status::OK(); -} - -template -void AdaptiveSharedBatchScheduler::AddBatch( - internal::ASBSBatch* batch) { - mutex_lock l(mu_); - batches_.push(batch); -} - -template -void AdaptiveSharedBatchScheduler::RemoveQueue( - const internal::ASBSQueue* queue) { - mutex_lock l(mu_); - queues_and_callbacks_.erase(queue); -} - -template -void AdaptiveSharedBatchScheduler::ProcessOneBatch() { - static const double kFeedbackMultiplier = .001; - internal::ASBSBatch* batch = nullptr; - BatchProcessor callback; - const int64 start_time_micros = GetEnv()->NowMicros(); - { - mutex_lock l(mu_); - if (!batches_.empty()) { - batch = batches_.top(); - batches_.pop(); - callback = queues_and_callbacks_[batch->queue()]; - } - } - if (batch != nullptr) { - double feedback = options_.scheduling_period_feedback(); - const int64 N = options_.feedback_smoothing_batches; - ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N; - scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_); - if (scheduling_period_ < options_.min_scheduling_period_micros) { - scheduling_period_ = options_.min_scheduling_period_micros; - } else if (scheduling_period_ > options_.max_scheduling_period_micros) { - scheduling_period_ = options_.max_scheduling_period_micros; - } - // Queue may destroy itself after ReleaseBatch is called. - batch->queue()->ReleaseBatch(batch); - batch_thread_pool_->Schedule([callback, batch] { - callback(std::unique_ptr>(batch)); - }); - } - const int64 sleep_time = - scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros); - if (sleep_time > 0) { - GetEnv()->SleepForMicroseconds(sleep_time); - } -} - -template -bool AdaptiveSharedBatchScheduler::BatchCompare::operator()( - const internal::ASBSBatch* a, - const internal::ASBSBatch* b) { - return a->creation_time_micros() > b->creation_time_micros(); -} - -// ---------------- ASBSQueue ---------------- - -namespace internal { -template -ASBSQueue::ASBSQueue( - std::shared_ptr> scheduler, - const QueueOptions& options) - : scheduler_(scheduler), options_(options) {} - -template -ASBSQueue::~ASBSQueue() { - // Wait until last batch has been scheduled. - const int kSleepMicros = 1000; - for (;;) { - { - mutex_lock l(mu_); - if (num_enqueued_batches_ == 0) { - break; - } - } - scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros); - } - scheduler_->RemoveQueue(this); -} - -template -Status ASBSQueue::Schedule(std::unique_ptr* task) { - bool added_new_batch = false; - size_t size = (*task)->size(); - if (size > options_.max_batch_size) { - return errors::InvalidArgument("Task size ", size, - " is larger than maximum batch size ", - options_.max_batch_size); - } - { - mutex_lock l(mu_); - // Current batch is full, create another if allowed. - if (current_batch_ && - current_batch_->size() + size > options_.max_batch_size) { - if (num_enqueued_batches_ >= options_.max_enqueued_batches) { - return errors::Unavailable("The batch scheduling queue is full"); - } - current_batch_->Close(); - current_batch_ = nullptr; - } - if (!current_batch_) { - added_new_batch = true; - num_enqueued_batches_++; - current_batch_ = - new ASBSBatch(this, scheduler_->GetEnv()->NowMicros()); - } - current_batch_->AddTask(std::move(*task)); - num_enqueued_tasks_++; - } - if (added_new_batch) scheduler_->AddBatch(current_batch_); - return Status::OK(); -} - -template -void ASBSQueue::ReleaseBatch(const ASBSBatch* batch) { - mutex_lock l(mu_); - num_enqueued_batches_--; - num_enqueued_tasks_ -= batch->num_tasks(); - if (batch == current_batch_) { - current_batch_->Close(); - current_batch_ = nullptr; - } -} - -template -size_t ASBSQueue::NumEnqueuedTasks() const { - mutex_lock l(mu_); - return num_enqueued_tasks_; -} - -template -size_t ASBSQueue::SchedulingCapacity() const { - mutex_lock l(mu_); - const int current_batch_capacity = - current_batch_ ? options_.max_batch_size - current_batch_->size() : 0; - const int spare_batches = - options_.max_enqueued_batches - num_enqueued_batches_; - return spare_batches * options_.max_batch_size + current_batch_capacity; -} -} // namespace internal -} // namespace serving -} // namespace tensorflow - -#endif // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc deleted file mode 100644 index a07cd6d834..0000000000 --- a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc +++ /dev/null @@ -1,438 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h" - -#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace serving { -namespace anonymous { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on -// that task. Returns the resulting status. -Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { - std::unique_ptr task(new FakeTask(task_size)); - Status status = scheduler->Schedule(&task); - // Schedule() should have consumed 'task' iff it returned Status::OK. - CHECK_EQ(status.ok(), task == nullptr); - return status; -} - -// Creates a thread that waits on 'start' and then advances the fake clock in -// 'env' in a loop until 'stop' is notified. Useful for allowing objects that -// use the clock to be destroyed. -std::unique_ptr CreateFakeClockAdvancerThread( - test_util::FakeClockEnv* env, Notification* start, Notification* stop) { - return std::unique_ptr(Env::Default()->StartThread( - {}, "FakeClockAdvancerThread", [env, start, stop] { - start->WaitForNotification(); - while (!stop->HasBeenNotified()) { - env->AdvanceByMicroseconds(10); - Env::Default()->SleepForMicroseconds(10); - } - })); -} - -TEST(AdaptiveSharedBatchSchedulerTest, Basic) { - for (const bool delete_scheduler_early : {false, true}) { - for (const bool delete_queue_1_early : {false, true}) { - int queue_0_tasks = 0; - auto queue_0_callback = - [&queue_0_tasks](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - for (int i = 0; i < batch->num_tasks(); i++) { - queue_0_tasks += batch->task(i).size(); - } - }; - int queue_1_tasks = 0; - auto queue_1_callback = - [&queue_1_tasks](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - for (int i = 0; i < batch->num_tasks(); i++) { - queue_1_tasks += batch->task(i).size(); - } - }; - { - std::shared_ptr> scheduler; - TF_ASSERT_OK( - AdaptiveSharedBatchScheduler::Create({}, &scheduler)); - - // Create two queues. - std::unique_ptr> queue_0; - TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0)); - std::unique_ptr> queue_1; - TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1)); - - if (delete_scheduler_early) { - // Delete our copy of the scheduler. The queues should keep it alive - // under the covers. - scheduler = nullptr; - } - // Submit tasks to the two queues, and (optionally) remove the queues. - TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); - TF_ASSERT_OK(ScheduleTask(3, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(4, queue_1.get())); - if (delete_queue_1_early) { - queue_1 = nullptr; - } - TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); - } - EXPECT_EQ(queue_0_tasks, 9); - EXPECT_EQ(queue_1_tasks, 6); - } - } -} - -TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) { - using Scheduler = AdaptiveSharedBatchScheduler; - std::shared_ptr scheduler; - Scheduler::Options options; - options.num_batch_threads = 0; - EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); - options = Scheduler::Options(); - options.min_scheduling_period_micros = 50; - options.max_scheduling_period_micros = 100; - options.initial_scheduling_period_micros = 1; - EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); - options = Scheduler::Options(); - options.min_scheduling_period_micros = 50; - options.max_scheduling_period_micros = 100; - options.initial_scheduling_period_micros = 1000; - EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); - options = Scheduler::Options(); - options.min_scheduling_period_micros = 100; - options.max_scheduling_period_micros = 50; - options.initial_scheduling_period_micros = 75; - EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); - options = Scheduler::Options(); - options.feedback_smoothing_batches = 0; - EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); -} - -TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - { - AdaptiveSharedBatchScheduler::Options options; - options.initial_scheduling_period_micros = 1000; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK( - AdaptiveSharedBatchScheduler::Create(options, &scheduler)); - std::unique_ptr> queue_0; - std::unique_ptr> queue_1; - int queue_0_tasks = 0; - int queue_1_tasks = 0; - auto queue_0_callback = [&queue_0_tasks, - &env](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - for (int i = 0; i < batch->num_tasks(); i++) { - queue_0_tasks += batch->task(i).size(); - } - env.SleepForMicroseconds(1); - }; - auto queue_1_callback = [&queue_1_tasks, - &env](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - for (int i = 0; i < batch->num_tasks(); i++) { - queue_1_tasks += batch->task(i).size(); - } - env.SleepForMicroseconds(1); - }; - AdaptiveSharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.max_enqueued_batches = 0; - // Queue must have max_enqueued_batchs > 1. - EXPECT_FALSE( - scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok()); - queue_options.max_enqueued_batches = 2; - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); - queue_options.max_batch_size = 0; - // Queue must have max_batch_size > 0. - EXPECT_FALSE( - scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok()); - queue_options.max_batch_size = 2; - queue_options.max_enqueued_batches = 1; - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); - - // Wait for scheduling_thread to sleep. - env.BlockUntilThreadsAsleep(1); - // Task larger than max_batch_size shouldn't schedule. - EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok()); - TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); - env.AdvanceByMicroseconds(1); - - // Task larger than max_batch_size shouldn't schedule. - EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok()); - TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); - TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); - env.AdvanceByMicroseconds(1); - // Exceeds max_enqueued_batches, shouldn't schedule. - EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok()); - - TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); - // Exceeds max_enqueued_batches, shouldn't schedule. - EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok()); - TF_ASSERT_OK(ScheduleTask(4, queue_0.get())); - - // Batches should be processed in order from oldest to newest. - env.AdvanceByMicroseconds(1000); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(queue_0_tasks, 10); - EXPECT_EQ(queue_1_tasks, 0); - - env.AdvanceByMicroseconds(1000); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(queue_0_tasks, 10); - EXPECT_EQ(queue_1_tasks, 2); - - env.AdvanceByMicroseconds(1000); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(queue_0_tasks, 19); - EXPECT_EQ(queue_1_tasks, 2); - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - { - double feedback = 0; - AdaptiveSharedBatchScheduler::Options options; - options.initial_scheduling_period_micros = 1000; - options.min_scheduling_period_micros = 200; - options.max_scheduling_period_micros = 2000; - options.env = &env; - options.scheduling_period_feedback = [&feedback] { return feedback; }; - options.feedback_smoothing_batches = 1; - std::shared_ptr> scheduler; - TF_ASSERT_OK( - AdaptiveSharedBatchScheduler::Create(options, &scheduler)); - std::unique_ptr> queue; - int scheduled_items = 0; - auto queue_callback = [&scheduled_items, - &env](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - scheduled_items = 0; - for (int i = 0; i < batch->num_tasks(); i++) { - scheduled_items += batch->task(i).size(); - } - env.SleepForMicroseconds(1); - }; - - TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); - - // Wait for scheduling_thread to sleep. - env.BlockUntilThreadsAsleep(1); - // Enqueue 6 batches. - for (int i = 0; i < 6; i++) { - TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); - env.AdvanceByMicroseconds(1); - } - feedback = -500; - env.AdvanceByMicroseconds(994); - env.BlockUntilThreadsAsleep(2); // scheduling period = 500 usec. - EXPECT_EQ(scheduled_items, 900); - env.AdvanceByMicroseconds(500); - env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. - EXPECT_EQ(scheduled_items, 901); - feedback = 0; - env.AdvanceByMicroseconds(250); - env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. - EXPECT_EQ(scheduled_items, 902); - feedback = 10000; // large feedback should hit max_scheduling_period. - env.AdvanceByMicroseconds(250); - env.BlockUntilThreadsAsleep(2); // scheduling period = 2000 usec. - EXPECT_EQ(scheduled_items, 903); - feedback = -10000; // large feedback should hit min_scheduling_period. - env.AdvanceByMicroseconds(1999); - // No callback scheduled, only scheduling thread sleeping. - env.BlockUntilThreadsAsleep(1); - EXPECT_EQ(scheduled_items, 903); - env.AdvanceByMicroseconds(1); - env.BlockUntilThreadsAsleep(2); // scheduling period = 200 usec. - EXPECT_EQ(scheduled_items, 904); - env.AdvanceByMicroseconds(200); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(scheduled_items, 905); - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - { - double feedback = 0; - AdaptiveSharedBatchScheduler::Options options; - options.initial_scheduling_period_micros = 1000; - options.env = &env; - options.scheduling_period_feedback = [&feedback] { return feedback; }; - options.feedback_smoothing_batches = 3; - std::shared_ptr> scheduler; - TF_ASSERT_OK( - AdaptiveSharedBatchScheduler::Create(options, &scheduler)); - std::unique_ptr> queue; - int scheduled_items = 0; - auto queue_callback = [&scheduled_items, - &env](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - scheduled_items = 0; - for (int i = 0; i < batch->num_tasks(); i++) { - scheduled_items += batch->task(i).size(); - } - env.SleepForMicroseconds(1); - }; - - TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); - - // Wait for scheduling_thread to sleep. - env.BlockUntilThreadsAsleep(1); - // Enqueue 4 batches. - for (int i = 0; i < 4; i++) { - TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); - env.AdvanceByMicroseconds(1); - } - feedback = -300; - env.AdvanceByMicroseconds(996); - env.BlockUntilThreadsAsleep(2); - // ewma_feedback = 100, scheduling_period = 900. - EXPECT_EQ(scheduled_items, 900); - env.AdvanceByMicroseconds(899); - // No callback scheduled, only scheduling thread sleeping. - env.BlockUntilThreadsAsleep(1); - EXPECT_EQ(scheduled_items, 900); - env.AdvanceByMicroseconds(1); - env.BlockUntilThreadsAsleep(2); - // ewma_feedback = 167, scheduling_period = 750. - EXPECT_EQ(scheduled_items, 901); - env.AdvanceByMicroseconds(749); - // No callback scheduled, only scheduling thread sleeping. - env.BlockUntilThreadsAsleep(1); - EXPECT_EQ(scheduled_items, 901); - feedback = 1000 / 3.; - env.AdvanceByMicroseconds(1); - env.BlockUntilThreadsAsleep(2); - // emwa_feedback = 0, scheduling_period = 750. - EXPECT_EQ(scheduled_items, 902); - env.AdvanceByMicroseconds(749); - // No callback scheduled, only scheduling thread sleeping. - env.BlockUntilThreadsAsleep(1); - EXPECT_EQ(scheduled_items, 902); - env.AdvanceByMicroseconds(1); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(scheduled_items, 903); - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - { - AdaptiveSharedBatchScheduler::Options options; - options.initial_scheduling_period_micros = 1000; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK( - AdaptiveSharedBatchScheduler::Create(options, &scheduler)); - std::unique_ptr> queue; - int scheduled_items = 0; - auto queue_callback = [&scheduled_items, - &env](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - EXPECT_GT(batch->num_tasks(), 0); - scheduled_items = 0; - for (int i = 0; i < batch->num_tasks(); i++) { - scheduled_items += batch->task(i).size(); - } - env.SleepForMicroseconds(1); - }; - AdaptiveSharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.max_enqueued_batches = 10; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue)); - - // Wait for scheduling_thread to sleep. - env.BlockUntilThreadsAsleep(1); - // Enqueue 3 tasks. - EXPECT_EQ(queue->NumEnqueuedTasks(), 0); - EXPECT_EQ(queue->SchedulingCapacity(), 100); - TF_ASSERT_OK(ScheduleTask(5, queue.get())); - EXPECT_EQ(queue->NumEnqueuedTasks(), 1); - EXPECT_EQ(queue->SchedulingCapacity(), 95); - env.AdvanceByMicroseconds(1); - TF_ASSERT_OK(ScheduleTask(6, queue.get())); - EXPECT_EQ(queue->NumEnqueuedTasks(), 2); - EXPECT_EQ(queue->SchedulingCapacity(), 84); - env.AdvanceByMicroseconds(1); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - EXPECT_EQ(queue->NumEnqueuedTasks(), 3); - EXPECT_EQ(queue->SchedulingCapacity(), 83); - - env.AdvanceByMicroseconds(998); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(scheduled_items, 5); - env.AdvanceByMicroseconds(1000); - env.BlockUntilThreadsAsleep(2); - EXPECT_EQ(scheduled_items, 7); - start_teardown.Notify(); - } - stop_teardown.Notify(); -} -} // namespace anonymous -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h index a5072f439a..7c41ad8818 100644 --- a/tensorflow/contrib/batching/batch_scheduler.h +++ b/tensorflow/contrib/batching/batch_scheduler.h @@ -78,7 +78,7 @@ template class Batch { public: Batch() = default; - virtual ~Batch(); // Blocks until the batch is closed. + ~Batch(); // Blocks until the batch is closed. // Appends 'task' to the batch. After calling AddTask(), the newly-added task // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1). -- GitLab From 37fd951790d7ad27c679c925c28b01ca73875738 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 20 Oct 2017 14:49:56 -0700 Subject: [PATCH 220/573] Simplifies capturing code in graph_callable to use recent function improvements. PiperOrigin-RevId: 172937003 --- tensorflow/python/eager/graph_callable.py | 57 +++---------------- .../python/ops/resource_variable_ops.py | 15 ++--- 2 files changed, 14 insertions(+), 58 deletions(-) diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index 0ec83636a0..7f7a8c4a88 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -45,28 +45,6 @@ def _default_initializer(name, shape, dtype): return initializer[0] -class _VariableFromResource(resource_variable_ops.ResourceVariable): - """Variable object from a preexisting resource. - - Required because the ResourceVariable constructor creates the resource handle, - and here we want to use a preexisting one. - """ - - def __init__(self, resource, dtype, name, shape): - self._handle = resource - self._graph_shape = tensor_shape.as_shape(shape) - self._handle_device = resource.device - self._handle_name = name - self._cached_value = None - self._initializer_op = None - self._caching_device = None - self._dtype = dtype - self._constraint = None - self._in_graph_mode = context.in_graph_mode() - if self._in_graph_mode: - self._graph_element = self.read_value() - - class _CapturedVariable(object): """Variable captured by graph_callable. @@ -137,17 +115,11 @@ class _VariableCapturingScope(object): trainable=True, collections=None, caching_device=None, # pylint: disable=redefined-outer-name partitioner=None, validate_shape=True, use_resource=None): - del getter, regularizer, partitioner, validate_shape, use_resource - del collections, initializer, trainable, reuse, caching_device + del getter, regularizer, partitioner, validate_shape, use_resource, dtype + del collections, initializer, trainable, reuse, caching_device, shape, assert name in self.variables v = self.variables[name] - v.placeholder = array_ops.placeholder(dtype=dtypes.resource, shape=shape) - # TODO(apassos) remove the need for this by correctly dealing with shape - # inference. - v.placeholder._handle_data = v.variable.handle._handle_data # pylint: disable=protected-access - return _VariableFromResource( - v.placeholder, dtype=dtypes.as_dtype(dtype), name=name, - shape=v.shape) + return v.variable scope = variable_scope.get_variable_scope() with variable_scope.variable_scope(scope, custom_getter=_custom_getter): @@ -181,14 +153,12 @@ class _VariableCapturingScope(object): v = _CapturedVariable(name, initializer, shape, dtype, trainable) self.variables[name] = v - graph_mode_resource = resource_variable_ops.var_handle_op( - shared_name=name, shape=shape, dtype=dtype) + graph_mode_resource = v.variable.handle if initializer is None: initializer = _default_initializer(name, shape, dtype) resource_variable_ops.assign_variable_op( graph_mode_resource, initializer(shape, dtype)) - return _VariableFromResource( - graph_mode_resource, dtype, name, shape=v.shape) + return v.variable scope = variable_scope.get_variable_scope() with variable_scope.variable_scope(scope, custom_getter=_custom_getter): @@ -220,13 +190,6 @@ class _FunctionObject(function._GraphModeFunction): # pylint: disable=protected def variables(self): return [x.variable for x in self._variables] - def __call__(self, *args, **kwds): - kwds.pop("want_gradients", False) - if kwds: - raise ValueError("graph_callable functions do not take keyword args") - values = [x.variable.handle for x in self._variables] - return super(_FunctionObject, self).__call__(*(values + list(args))) - class _InitializingFunctionObject(object): """Responsible for deciding which version of func-to-object to call. @@ -318,7 +281,8 @@ def _graph_callable_internal(func, shape_and_dtypes): # This graph will store both the initialization and the call version of the # wrapped function. It will later be used by the backprop code to build the # backprop graph, if necessary. - tmp_graph = tf_ops.Graph() + captures = {} + tmp_graph = function.CapturingGraph(captures) # Inherit the container from the original graph to create resources at user # expected containers. Also inherits the container prefix, since this is # used for error checking when isolating Eager execution (the container @@ -342,7 +306,6 @@ def _graph_callable_internal(func, shape_and_dtypes): # variables. As a side-effect this will populate the variable capturing # scope's view of which variables exist. variable_captures = _VariableCapturingScope() - captures = {} with variable_captures.initializing_scope(), function.capture_tensors( captures): func_outputs = func(*func_inputs) @@ -366,7 +329,6 @@ def _graph_callable_internal(func, shape_and_dtypes): sorted_variables = sorted(variable_captures.variables.values(), key=lambda x: x.name) - variable_placeholders = [x.placeholder for x in sorted_variables] ids = list(sorted(captures.keys())) if ids: extra_inputs, extra_placeholders = zip(*[captures[x] for x in ids]) @@ -377,7 +339,6 @@ def _graph_callable_internal(func, shape_and_dtypes): flat_inputs = [x for x in nest.flatten(func_inputs) if isinstance(x, tf_ops.Tensor)] placeholder_inputs = flat_inputs+ list(extra_placeholders) - all_inputs = variable_placeholders + placeholder_inputs func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)] initializer_function_def = function.make_function_def( @@ -407,13 +368,13 @@ def _graph_callable_internal(func, shape_and_dtypes): captured_function_def = function.make_function_def( tmp_graph, capturing_operations, - all_inputs, + placeholder_inputs, capture_func_def_outputs) function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access captured_function_def) captured_function = _FunctionObject( sorted_variables, - all_inputs, + placeholder_inputs, extra_inputs, captured_function_def, tmp_graph, diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index c94ddb0627..71e1fb0297 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -26,7 +26,6 @@ from tensorflow.python.eager import tape from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_resource_variable_ops @@ -315,7 +314,7 @@ class ResourceVariable(variables.Variable): self._handle_device = ( self._handle.device if self._in_graph_mode else context.get_default_context().device_name) - self._graph_shape = initial_value.get_shape() + self._shape = initial_value.get_shape() else: initial_value = initial_value() with ops.name_scope("Initializer"): @@ -330,7 +329,7 @@ class ResourceVariable(variables.Variable): self._handle_device = ( self._handle.device if self._in_graph_mode else context.get_default_context().device_name) - self._graph_shape = initial_value.get_shape() + self._shape = initial_value.get_shape() # pylint: enable=protected-access # Or get the initial value from a Tensor or Python object. @@ -355,7 +354,7 @@ class ResourceVariable(variables.Variable): graph_mode=self._in_graph_mode) self._handle_device = (self._handle.device if self._in_graph_mode else context.get_default_context().device_name) - self._graph_shape = initial_value.get_shape() + self._shape = initial_value.get_shape() self._initial_value = initial_value if self._in_graph_mode else None self._handle_name = handle_name + ":0" @@ -422,7 +421,7 @@ class ResourceVariable(variables.Variable): self._handle = g.as_graph_element( ops.prepend_name_scope( variable_def.variable_name, import_scope=import_scope)) - self._graph_shape = tensor_shape.TensorShape( + self._shape = tensor_shape.TensorShape( self._handle.op.get_attr("shape")) self._handle_device = self._handle.device self._handle_name = self._handle.name @@ -502,11 +501,7 @@ class ResourceVariable(variables.Variable): @property def shape(self): """The shape of this variable.""" - if self._in_graph_mode: - return self._graph_shape - return tensor_shape.TensorShape( - tensor_util.constant_value( - gen_resource_variable_ops.variable_shape(self._handle))) + return self._shape @property def create(self): -- GitLab From 41df2cec28274cff4538f4735202471f8da45ce8 Mon Sep 17 00:00:00 2001 From: ashankar Date: Fri, 20 Oct 2017 14:06:54 -0800 Subject: [PATCH 221/573] Testing pending CL: 172939383 --- tensorflow/contrib/eager/README.OPENSOURCE.md | 15 + tensorflow/contrib/eager/README.md | 74 +- .../contrib/eager/python/examples/BUILD | 134 +++ .../eager/python/examples/cart_pole.py | 282 ++++++ .../eager/python/examples/cart_pole_helper.py | 60 ++ .../python/examples/linear_regression.py | 197 ++++ .../python/examples/notebooks/1_basics.ipynb | 529 +++++++++++ .../examples/notebooks/2_gradients.ipynb | 864 ++++++++++++++++++ .../examples/notebooks/3_datasets.ipynb | 218 +++++ .../examples/tests/cart_pole_helper_test.py | 51 ++ .../python/examples/tests/cart_pole_test.py | 162 ++++ .../examples/tests/linear_regression_test.py | 114 +++ .../eager/python/examples/tests/spinn_test.py | 311 +++++++ 13 files changed, 2999 insertions(+), 12 deletions(-) create mode 100644 tensorflow/contrib/eager/README.OPENSOURCE.md create mode 100644 tensorflow/contrib/eager/python/examples/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/cart_pole.py create mode 100644 tensorflow/contrib/eager/python/examples/cart_pole_helper.py create mode 100644 tensorflow/contrib/eager/python/examples/linear_regression.py create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py create mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py create mode 100644 tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py create mode 100644 tensorflow/contrib/eager/python/examples/tests/spinn_test.py diff --git a/tensorflow/contrib/eager/README.OPENSOURCE.md b/tensorflow/contrib/eager/README.OPENSOURCE.md new file mode 100644 index 0000000000..a4a3af08cf --- /dev/null +++ b/tensorflow/contrib/eager/README.OPENSOURCE.md @@ -0,0 +1,15 @@ +TensorFlow has many kernels for doing (deep) learning and data manipulation. +There are typically assembled into computational graphs which can run +efficiently in a variety of environments. + +We are exploring an alternative interaction, where kernels are invoked +immediately and call this "eager execution". We are hoping to retain the +benefits of graphs while improving usability with benefits like: + +- Immediate error messages and easier debugging +- Flexibility to use Python datastructures and control flow +- Reduced boilerplate + +Eager execution is under active development. +There are not many developer-facing materials yet, but stay tuned for updates +in this directory. diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md index a4a3af08cf..fe577fa7eb 100644 --- a/tensorflow/contrib/eager/README.md +++ b/tensorflow/contrib/eager/README.md @@ -1,15 +1,65 @@ -TensorFlow has many kernels for doing (deep) learning and data manipulation. -There are typically assembled into computational graphs which can run -efficiently in a variety of environments. +# TensorFlow Eager Execution -We are exploring an alternative interaction, where kernels are invoked -immediately and call this "eager execution". We are hoping to retain the -benefits of graphs while improving usability with benefits like: +> *WARNING*: This is a preview/pre-alpha version. The API and performance +> characteristics are subject to change. -- Immediate error messages and easier debugging -- Flexibility to use Python datastructures and control flow -- Reduced boilerplate -Eager execution is under active development. -There are not many developer-facing materials yet, but stay tuned for updates -in this directory. +Eager execution is an experimental interface to TensorFlow that provides an +imperative programming style (à la [NumPy](http://www.numpy.org)). When you +enable eager execution, TensorFlow operations execute immediately; you do not +execute a pre-constructed graph with +[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session). + +For example, consider a simple computation in TensorFlow: + +```python +x = tf.placeholder(tf.float32, shape=[1, 1]) +m = tf.matmul(x, x) + +with tf.Session() as sess: + print(sess.run(m, feed_dict={x: [[2.]]})) + +# Will print [[4.]] +``` + +Eager execution makes this much simpler: + +```python +x = [[2.]] +m = tf.matmul(x, x) + +print(m) +``` + +## Installation + +Since eager execution is not yet part of a TensorFlow release, using it requires +either [building from source](https://www.tensorflow.org/install/install_sources) +or the latest nightly builds. The nightly builds are available as: + +- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and + +- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images. + +For example, to run the latest nightly docker image: + +```sh +# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker +nvidia-docker pull tensorflow/tensorflow:nightly-gpu +nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu + +# If you do not have a GPU, use the CPU-only image +docker pull tensorflow/tensorflow:nightly +docker run -it -p 8888:8888 tensorflow/tensorflow:nightly +``` + +And then visit http://localhost:8888 in your browser for a Jupyter notebook +environment. Try out the notebooks below. + +## Documentation + +For an introduction to TensorFlow eager execution, see the Jupyter notebooks: + +- [Basic Usage](examples/notebooks/1_basics.ipynb) +- [Gradients](examples/notebooks/2_gradients.ipynb) +- [Importing Data](examples/notebooks/3_datasets.ipynb) diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD new file mode 100644 index 0000000000..3604139819 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/BUILD @@ -0,0 +1,134 @@ +# Description: +# Open-source examples and tutorials for TensorFlow Eager Execution. + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +load("//tensorflow:tensorflow.bzl", "py_test") +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "linear_regression", + srcs = ["linear_regression.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "@six_archive//:six", + ], +) + +py_library( + name = "cart_pole_helper", + srcs = ["cart_pole_helper.py"], + srcs_version = "PY2AND3", +) + +py_library( + name = "spinn", + srcs = ["spinn.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//third_party/py/numpy", + "@six_archive//:six", + ], +) + +py_binary( + name = "cart_pole", + srcs = ["cart_pole.py"], + srcs_version = "PY2AND3", + deps = [ + ":cart_pole_helper", + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "@six_archive//:six", + ], +) + +py_binary( + name = "spinn_prep_data", + srcs = ["spinn_prep_data.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + ], +) + +py_binary( + name = "spinn_train", + srcs = ["spinn_train.py"], + srcs_version = "PY2AND3", + deps = [ + ":spinn", + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "//third_party/py/numpy", + "@six_archive//:six", + ], +) + +cuda_py_test( + name = "linear_regression_test", + size = "small", + srcs = ["tests/linear_regression_test.py"], + additional_deps = [ + ":linear_regression", + "//tensorflow/python/eager:test", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + ], +) + +py_test( + name = "cart_pole_helper_test", + srcs = ["tests/cart_pole_helper_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":cart_pole_helper", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + ], +) + +cuda_py_test( + name = "cart_pole_test", + size = "small", + srcs = ["tests/cart_pole_test.py"], + additional_deps = [ + ":cart_pole", + "//tensorflow/python/eager:test", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:training", + ], +) + +cuda_py_test( + name = "spinn_test", + size = "medium", + srcs = ["tests/spinn_test.py"], + additional_deps = [ + ":spinn", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + "//tensorflow/python/eager:test", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/contrib/eager/python/examples/cart_pole.py b/tensorflow/contrib/eager/python/examples/cart_pole.py new file mode 100644 index 0000000000..56235e4039 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/cart_pole.py @@ -0,0 +1,282 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""TensorFlow Eager Execution Example: OpenAI Gym CartPole. + +Solves the cart-pole problem with policy gradient-based reinforcement learning. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import sys + +import gym +import numpy as np +from six.moves import input # pylint: disable=redefined-builtin +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.contrib.eager.python.examples import cart_pole_helper + + +class PolicyNetwork(object): + """Policy network for the cart-pole reinforcement learning problem. + + The forward path of the network takes an observation from the cart-pole + environment (length-4 vector) and outputs an action. + """ + + def __init__(self, hidden_size, train_logdir=None): + """Constructor of PolicyNetwork. + + Args: + hidden_size: Size of the hidden layer, as an `int`. + train_logdir: The directory in which summaries will be written for + TensorBoard during training (optional). + """ + self._hidden_layer = tf.layers.Dense(hidden_size, activation=tf.nn.elu) + self._output_layer = tf.layers.Dense(1) + + # Gradient function. + self._grad_fn = tfe.implicit_gradients( + self._get_cross_entropy_and_save_actions) + + # Support for TensorBoard summaries. Once training has started, use: + # tensorboard --logdir= + self._summary_writer = (tfe.SummaryWriter(train_logdir) if train_logdir + else None) + + def forward(self, inputs): + """Given inputs, calculate logits and action. + + Args: + inputs: Observations from a step in the cart-pole environment, of shape + `(batch_size, input_size)` + + Returns: + logits: the logits output by the output layer. This can be viewed as the + likelihood vales of choosing the left (0) action. Shape: + `(batch_size, 1)`. + actions: randomly selected actions ({0, 1}) based on the logits. Shape: + `(batch_size, 1)`. + """ + hidden = self._hidden_layer(inputs) + logits = self._output_layer(hidden) + + # Probability of selecting the left action. + left_p = tf.nn.sigmoid(logits) + # Probabilities of selecting the left and right actions. + left_right_ps = tf.concat([left_p, 1.0 - left_p], 1) + # Randomly-generated actions based on the probabilities. + actions = tf.multinomial(tf.log(left_right_ps), 1) + return logits, actions + + def _get_cross_entropy_and_save_actions(self, inputs): + """Given inputs, get the sigmoid cross entropy and save selection action. + + Args: + inputs: Observation from a step in the cart-pole environment. + + Returns: + The sigmoid cross-entropy loss given the selected action and logits, based + on the assumption that the selected action was rewarded by the + environment. + """ + logits, actions = self.forward(inputs) + + # N.B.: This is an important step. We save the value of the `actions` in a + # member variable for use with the RL environment. In classic TensorFlow + # (non-eager execution), it is less straightfoward to access intermediate + # computation results in this manner (c.f., `tf.Session.partial_run()`). + self._current_actions = actions + + labels = 1.0 - tf.cast(actions, tf.float32) + return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) + + def train(self, + cart_pole_env, + optimizer, + discount_rate, + num_games, + max_steps_per_game): + """Train the PolicyNetwork by playing `num_games` games in `cart_pole_env`. + + Arguments: + cart_pole_env: The cart-pole gym environment object. + optimizer: A TensorFlow `Optimizer` object to be used in this training + (e.g., `tf.train.AdamOptimizer`). + discount_rate: Reward discounting rate. + num_games: Number of games to run per parameter update. + max_steps_per_game: Maximum number of steps to run in each game. + + Returns: + Step counts from all games, as a `list` of `int`. + """ + all_gradient_lists = [] + all_rewards = [] + for _ in xrange(num_games): + obs = cart_pole_env.reset() + game_rewards = [] + game_gradient_lists = [] + for _ in xrange(max_steps_per_game): + # TODO(cais): Can we save the tf.constant() call? + grad_list, var_list = zip(*self._grad_fn(tf.constant([obs]))) + game_gradient_lists.append(grad_list) + + action = self._current_actions.numpy()[0][0] + obs, reward, done, _ = cart_pole_env.step(action) + game_rewards.append(reward) + if reward != 1.0 or done: + break + + all_gradient_lists.append(game_gradient_lists) + all_rewards.append(game_rewards) + + normalized_rewards = cart_pole_helper.discount_and_normalize_rewards( + all_rewards, discount_rate) + all_grads_and_vars = self._scale_and_average_gradients(var_list, + all_gradient_lists, + normalized_rewards) + optimizer.apply_gradients(all_grads_and_vars) + step_counts = [len(rewards) for rewards in all_rewards] + + if self._summary_writer: + self._summary_writer.scalar("mean_step_count", np.mean(step_counts)) + self._summary_writer.step() + + return step_counts + + def _scale_and_average_gradients(self, + variable_list, + all_gradient_lists, + normalized_rewards): + """Scale gradient tensors with normalized rewards.""" + num_games = len(all_gradient_lists) + grads_and_vars = [] + for j, var in enumerate(variable_list): + scaled_gradients = [] + for g in xrange(int(num_games)): + num_steps = len(all_gradient_lists[g]) + for s in xrange(num_steps): + scaled_gradients.append( + all_gradient_lists[g][s][j] * normalized_rewards[g][s]) + mean_scaled_gradients = sum(scaled_gradients) / len(scaled_gradients) + grads_and_vars.append((mean_scaled_gradients, var)) + return grads_and_vars + + def play(self, cart_pole_env, max_steps=None, render=False): + """Play a game in the cart-pole gym environment. + + Args: + cart_pole_env: The cart-pole gym environment object. + max_steps: Maximum number of steps to run in the game. + render: Whether the game state is to be rendered on the screen. + """ + if render: + input("\nAbout to play a game with rendering. Press Enter to continue: ") + + steps = 0 + obs = cart_pole_env.reset() + while True: + # TODO(cais): Can we save the tf.constant() call? + _, actions = self.forward(tf.constant([obs])) + if render: + cart_pole_env.render() + obs, reward, done, _ = cart_pole_env.step(actions.numpy()[0][0]) + steps += 1 + if done or reward != 1.0 or max_steps is not None and steps >= max_steps: + break + + +def main(_): + tf.set_random_seed(0) + + cart_pole_env = gym.make("CartPole-v0") + cart_pole_env.seed(0) + cart_pole_env.reset() + + device = "gpu:0" if tfe.num_gpus() else "cpu:0" + print("Using device: %s" % device) + + with tf.device(device): + policy_network = PolicyNetwork(FLAGS.hidden_size, train_logdir=FLAGS.logdir) + optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate) + + # Training loop. + for i in xrange(FLAGS.num_iterations): + step_counts = policy_network.train( + cart_pole_env, + optimizer, + FLAGS.discount_rate, + FLAGS.num_games_per_iteration, + FLAGS.max_steps_per_game) + print("Iteration %d: step counts = %s; mean = %g" % ( + i, step_counts, np.mean(step_counts))) + sys.stdout.flush() + + # Optional playing after training, with rendering. + if FLAGS.play_after_training: + policy_network.play(cart_pole_env, + max_steps=FLAGS.max_steps_per_game, + render=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--hidden_size", + type=int, + default=5, + help="Size of the hidden layer of the policy network.") + parser.add_argument( + "--discount_rate", + type=float, + default=0.95, + help="Reward discounting rate.") + parser.add_argument( + "--learning_rate", + type=float, + default=0.05, + help="Learning rate to be used during training.") + parser.add_argument( + "--num_iterations", + type=int, + default=100, + help="Number of training iterations.") + parser.add_argument( + "--num_games_per_iteration", + type=int, + default=20, + help="Number of games to run in each training iteration.") + parser.add_argument( + "--max_steps_per_game", + type=int, + default=1000, + help="Maximum number of steps to run in each game.") + parser.add_argument( + "--logdir", + type=str, + default=None, + help="logdir in which TensorBoard summaries will be written (optional).") + parser.add_argument( + "--play_after_training", + action="store_true", + help="Play a game after training (with rendering).") + + FLAGS, unparsed = parser.parse_known_args() + tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/cart_pole_helper.py b/tensorflow/contrib/eager/python/examples/cart_pole_helper.py new file mode 100644 index 0000000000..1b80f90165 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/cart_pole_helper.py @@ -0,0 +1,60 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helper functions for reinforcement learning in the cart-pole problem.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + + +def discount_rewards(rewards, discount_rate): + """Discout reward values with discount rate. + + Args: + rewards: A sequence of reward values in time. + discount_rate: (`float`) reward discounting rate (e.g., 0.95). + + Returns: + Discounted reward values. + """ + discounted = [] + for reward in reversed(rewards): + discounted.append( + (discounted[-1] if discounted else 0.0) * discount_rate + reward) + return list(reversed(discounted)) + + +def discount_and_normalize_rewards(reward_sequences, discount_rate): + """Perform discounting on a number of reward sequences; then normalize values. + + Args: + reward_sequences: an `iterable` of reward sequences. + discount_rate: reward discounting rate (e.g., 0.95). + + Returns: + A `list` of reward value `list`s, discounted and normalized. + """ + discounted = [] + for sequence in reward_sequences: + discounted.append(discount_rewards(sequence, discount_rate)) + discounted = np.array(discounted) + + # Compute overall mean and stddev. + flattened = np.concatenate(discounted) + mean = np.mean(flattened) + std = np.std(flattened) + return [((d - mean) / std) for d in discounted] diff --git a/tensorflow/contrib/eager/python/examples/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression.py new file mode 100644 index 0000000000..538d6d4225 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression.py @@ -0,0 +1,197 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=line-too-long +r"""TensorFlow Eager Execution Example: Linear Regression. + +This example shows how to use TensorFlow Eager Execution to fit a simple linear +regression model using some synthesized data. Specifically, it illustrates how +to define the forward path of the linear model and the loss function, as well +as how to obtain the gradients of the loss function with respect to the +variables and update the variables with the gradients. +""" +# pylint: enable=line-too-long + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import sys + +import numpy as np +from six.moves import xrange # pylint: disable=redefined-builtin +import tensorflow as tf + +# TODO(cais): Use tf.contrib.eager namespace when ready. +from tensorflow.contrib.eager.python import tfe + + +class DataGenerator(object): + """Generates synthetic data for linear regression.""" + + def __init__(self, w, b, noise_level, batch_size): + self._w = w + self._b = b + self._noise_level = noise_level + self._batch_size = batch_size + self._ndims = w.shape[0] + + def next_batch(self): + """Generate a synthetic batch of xs and ys.""" + xs = tf.random_normal([self._batch_size, self._ndims]) + ys = (tf.matmul(xs, self._w) + self._b + + self._noise_level * tf.random_normal([self._batch_size, 1])) + return xs, ys + + +class LinearModel(object): + """A TensorFlow linear regression model. + + Uses TensorFlow's eager execution. + + For those familiar with TensorFlow graphs, notice the absence of + `tf.Session`. The `forward()` method here immediately executes and + returns output values. The `loss()` method immediately compares the + output of `forward()` with the target adn returns the MSE loss value. + The `fit()` performs gradient-descent training on the model's weights + and bias. + """ + + def __init__(self): + """Constructs a LinearModel object.""" + self._hidden_layer = tf.layers.Dense(1) + + # loss_value_and_grad_fn is a function that when invoked, will return the + # loss value and the gradients of loss with respect to the variables. It has + # the same input arguments as `self.loss()`. + self._loss_value_and_grad_fn = tfe.implicit_value_and_gradients(self.loss) + + @property + def weights(self): + """Get values of weights as a numpy array.""" + return self._hidden_layer.variables[0].read_value().numpy() + + @property + def biases(self): + """Get values of biases as a numpy array.""" + return self._hidden_layer.variables[1].read_value().numpy() + + def forward(self, xs): + """Invoke the linear model. + + Args: + xs: input features, as a tensor of size [batch_size, ndims]. + + Returns: + ys: the predictions of the linear mode, as a tensor of size [batch_size] + """ + # Note: Unlike classic TensorFlow, operations such as self._hidden_layer + # will execute the underlying computation immediately. + return self._hidden_layer(xs) + + def loss(self, xs, ys): + """Loss of the linear model. + + Args: + xs: input features, as a tensor of size [batch_size, ndims]. + ys: the target values of y, as a tensor of size [batch_size]. + + Returns: + The mean square error loss value. + """ + return tf.reduce_mean(tf.square(self.forward(xs) - ys)) + + def fit(self, + batch_fn, + optimizer, + num_iters, + verbose=False, + logdir=None): + """Fit the linear-regression model. + + Args: + batch_fn: A function, which when called without any arguments, returns a + batch of xs and ys for training. + optimizer: The TensorFlow Optimizer object to be used. + num_iters: Number of training iterations to perform. + verbose: If true, will print out loss values at every iteration. + logdir: The directory in which summaries will be written for TensorBoard + (optional). + """ + if logdir: + # Support for TensorBoard summaries. Once training has started, use: + # tensorboard --logdir= + summary_writer = tfe.SummaryWriter(logdir) + + # Training loop. + for i in xrange(num_iters): + # Generate a (mini-)batch of data for training. + xs, ys = batch_fn() + + # Call the function obtained above to get the loss and gradient values at + # the specific training batch. The function has the same input arguments + # as the forward function, i.e., `linear_loss()`. + loss_value, grads_and_vars = self._loss_value_and_grad_fn(xs, ys) + if verbose: + print("Iteration %d: loss = %s" % (i, loss_value.numpy())) + + # Send the gradients to the optimizer and update the Variables, i.e., `w` + # and `b`. + optimizer.apply_gradients(grads_and_vars) + + if logdir: + summary_writer.scalar("loss", loss_value) + summary_writer.step() + + +def main(_): + # Ground-truth constants. + true_w = np.array([[-2.0], [4.0], [1.0]], dtype=np.float32) + true_b = np.array([0.5], dtype=np.float32) + noise_level = 0.01 + + # Training constants. + batch_size = 64 + learning_rate = 0.1 + num_iters = 20 + + print("True w: %s" % true_w) + print("True b: %s\n" % true_b) + + device = "gpu:0" if tfe.num_gpus() else "cpu:0" + print("Using device: %s" % device) + with tf.device(device): + linear_model = LinearModel() + + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + data_gen = DataGenerator(true_w, true_b, noise_level, batch_size) + linear_model.fit(data_gen.next_batch, optimizer, num_iters, verbose=True, + logdir=FLAGS.logdir) + + print("\nAfter training: w = %s" % linear_model.weights) + print("\nAfter training: b = %s" % linear_model.biases) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--logdir", + type=str, + default=None, + help="logdir in which TensorBoard summaries will be written (optional).") + FLAGS, unparsed = parser.parse_known_args() + + # Use tfe.run() instead of tf.app.run() for eager execution. + tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb new file mode 100644 index 0000000000..9c2e6f15b4 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb @@ -0,0 +1,529 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U9i2Dsh-ziXr" + }, + "source": [ + "# Eager Execution Tutorial: Basics\n", + "\n", + "This notebook introduces the basics of using TensorFlow's eager execution capabilities. It covers concepts such as:\n", + "\n", + "* Importing required packages\n", + "* Enabling eager execution\n", + "* Creating and using TensorFlow Tensors and Variables\n", + "* Using TensorFlow interactively\n", + "* Using GPUs with eager execution enabled\n", + "\n", + "This notebook does *not* cover modeling topics, such as gradients." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z1JcS5iBXMRO" + }, + "source": [ + "# Step 1: Import Eager\n", + "\n", + "The key imports for eager execution are the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RlIWhyeLoYnG" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "from tensorflow.contrib.eager.python import tfe" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H9UySOPLXdaw" + }, + "source": [ + "# Step 2: Enable eager execution\n", + "\n", + "All future TensorFlow calls will execute the\n", + "underlying TensorFlow ops immediately:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "WPTUfGq6kJ5w" + }, + "outputs": [], + "source": [ + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "twBfWd5xyu_d" + }, + "source": [ + "# Step 3: Interactively Use TensorFlow!\n", + "\n", + "Now you can call TensorFlow functions and get results, immediately! No more `tf.Sessions`!\n", + "\n", + "TensorFlow will automatically wrap native Python types for you with operator overloading for TensorFlow Tensors." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ngUe237Wt48W" + }, + "outputs": [], + "source": [ + "print(tf.add(1, 2))\n", + "print(tf.add([1, 2], [3, 4]))\n", + "print(tf.square(5))\n", + "print(tf.reduce_sum([1, 2, 3]))\n", + "print(tf.encode_base64(\"hello world\"))\n", + "print(\"\")\n", + "\n", + "x = tf.constant(2)\n", + "y = tf.constant(3)\n", + "print(x * y + 1)\n", + "\n", + "# Most TensorFlow ops are directly usable with eager execution, giving\n", + "# results immediately.\n", + "print(tf.contrib.signal.hamming_window(x * y + 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IDY4WsYRhP81" + }, + "source": [ + "Numpy arrays are supported, too:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "lCUWzso6mbqR" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "ones = np.ones([3, 3])\n", + "\n", + "print(\"numpy 3x3 matrix of 1s:\")\n", + "print(ones)\n", + "print(\"\")\n", + "\n", + "print(\"Multiplied by 42:\")\n", + "print(tf.multiply(ones, 42))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PBNP8yTRfu_X" + }, + "source": [ + "# Step 4: Define and Print TensorFlow Variables\n", + "\n", + "To define TensorFlow variables, use the `get_variable()` function as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "3Twf_Rw-gQFM" + }, + "outputs": [], + "source": [ + "x = tf.get_variable(name=\"x\", shape=[1], dtype=tf.float32, initializer=tf.zeros_initializer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "45G7094TxsMb" + }, + "source": [ + "## Printing TensorFlow Variables" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "UJBJeZ5XxuwA" + }, + "outputs": [], + "source": [ + "# This does NOT print the Variable's actual value:\n", + "print(\"Printing a TensorFlow Variable:\")\n", + "print(x)\n", + "print(\"\")\n", + "\n", + "# A TensorFlow variable represents a reference to a tensor.\n", + "# The `read_value()` method provides access to the current value of the\n", + "# variable. Tensorflow Variables are automatically initialized according to the\n", + "# semantics defined in tf.get_variable().\n", + "print(\"Printing a TensorFlow Variable's value using .read_value():\")\n", + "print(x.read_value())\n", + "print(\"\")\n", + "\n", + "print(\"Printing a TensorFlow Variable's value using .read_value().numpy():\")\n", + "print(x.read_value().numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "2njjWHcTpBEn" + }, + "source": [ + "## Changing a TensorFlow Variable's value\n", + "\n", + "To change a TensorFlow Variable's value, use its `.assign()` or `.assign_add()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "v3wr6Erbo_hB" + }, + "outputs": [], + "source": [ + "x.assign(42)\n", + "print(x.read_value())\n", + "\n", + "x.assign_add(3)\n", + "print(x.read_value())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "uhtynjHVpTB5" + }, + "source": [ + "## Use a Variable just like any other Tensor" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "7PbktdnHoehR" + }, + "outputs": [], + "source": [ + "print(x + 3)\n", + "\n", + "# This code will broadcast the value across the list of numbers:\n", + "print(x * [1, 2, 4])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GVChqwlwy1SI" + }, + "source": [ + "# Step 5: Debug Errors with Instant Feedback\n", + "\n", + "TensorFlow's eager execution helps you identify and debug runtime issues through interactive exploration of code snippets.\n", + "\n", + "Below, we'll define a length-4 vector, and attempt two `tf.slice()` operations,\n", + "one being legal and the other being illegal, leading to a runtime error that is\n", + "raised immediately." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "23ap04N0v4k0" + }, + "outputs": [], + "source": [ + "vector = tf.constant([10.0, 20.0, 30.0, 40.0])" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "FCUMsIYxxRRa" + }, + "outputs": [], + "source": [ + "# Works, because the values of `begin` and `size` (the 2nd and 3rd input\n", + "# arguments) are within the bound of `vector`.\n", + "print(tf.slice(vector, [1], [3]))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "T8me2oCNxpFp" + }, + "outputs": [], + "source": [ + "# The following does NOT work, because the value of `size` (the 3rd\n", + "# argument) causes the indices to go out of the bounds of `vector`. The\n", + "# error is raised immediately.\n", + "try:\n", + " print(tf.slice(vector, [1], [4]))\n", + "except tf.OpError as e:\n", + " print(\"Caught error: %s\" % e)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "irxJhAgar84v" + }, + "source": [ + "# Step 6: Using the GPU\n", + "\n", + "You can place Tensors on the GPU by calling a Tensor's `.gpu()` method.\n", + "\n", + "The first operation executing on the GPU may be slow as TensorFlow initializes. Subsequent uses will be much faster." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "7J4N9baqaKCL" + }, + "outputs": [], + "source": [ + "# The example code from here on will work only if your notebook\n", + "# is running on a machine with a functional CUDA GPU. The following\n", + "# line checks that.\n", + "is_gpu_available = tfe.num_gpus() \u003e 0\n", + "\n", + "# Create some Tensors\n", + "SIZE = 1000\n", + "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", + "\n", + "if is_gpu_available:\n", + " gpu_tensor = cpu_tensor.gpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "4E-2n7VbzY1n" + }, + "outputs": [], + "source": [ + "# Time a CPU-based matrix multiplication\n", + "\n", + "print(\"Time to conduct matmul on CPU:\")\n", + "%time tf.matmul(cpu_tensor, cpu_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "vbSFW-T5zhZF" + }, + "outputs": [], + "source": [ + "# Time GPU-based matrix multiplications.\n", + "\n", + "if is_gpu_available:\n", + " # First use of the GPU will be slow:\n", + " print(\"Time to conduct first matmul on GPU:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)\n", + " print()\n", + "\n", + " # Subsequent uses are much faster:\n", + " print(\"Time to conduct second matmul on GPU:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "E5pIOe3Rz7iW" + }, + "outputs": [], + "source": [ + "# Second timing demo for GPUs, after it has been used once:\n", + "\n", + "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", + "print(\"Time to conduct CPU matmul:\")\n", + "%time tf.matmul(cpu_tensor, cpu_tensor)\n", + "print()\n", + "\n", + "if is_gpu_available:\n", + " gpu_tensor = cpu_tensor.gpu()\n", + " print(\"Time to conduct GPU matmul:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "name": "Eager Execution Tutorial: Basics", + "provenance": [ + { + "file_id": "0B0kLcpwLFwKEVm9XNkFueGk4bTg", + "timestamp": 1504118841551 + } + ], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb new file mode 100644 index 0000000000..5e0ec5cf8a --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb @@ -0,0 +1,864 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "vDJ4XzMqodTy" + }, + "source": [ + "# Eager Execution: Working with Gradients\n", + "\n", + "This notebook demonstrates:\n", + "\n", + "* How to get gradients using TensorFlow's eager execution capabilities\n", + "* How to apply the gradients so you can update your variables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GQJysDM__Qb0" + }, + "source": [ + "# Setup: Import eager and enable eager execution.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "OiMPZStlibBv" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "from tensorflow.contrib.eager.python import tfe\n", + "\n", + "# Enable eager execution.\n", + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1CLWJl0QliB0" + }, + "source": [ + "# Fitting a Simple Linear Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-39gouo7mtgu" + }, + "source": [ + "## Step 1: Synthesize some data\n", + "\n", + "To demonstrate fitting a model with TensorFlow's eager execution, we'll fit a linear model to some synthesized data (which includes some noise).\n", + "\n", + "In the code, we use the variable names `w` and `b` to represent the single weight and bias we'll use to fit our model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "rQsdCg9PfIL-" + }, + "outputs": [], + "source": [ + "# The constants we'll try to fit our variables to:\n", + "true_w = 3\n", + "true_b = 2\n", + "\n", + "NUM_EXAMPLES = 1000\n", + "\n", + "# Our inputs:\n", + "inputs = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", + "\n", + "# Our labels, with noise:\n", + "noise = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", + "labels = inputs * true_w + true_b + noise" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 360, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 127, + "status": "ok", + "timestamp": 1505502830690, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "O4lsC4ckAcar", + "outputId": "2f760690-cafb-4777-b970-91d839f99faf" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAFXCAYAAACC+2avAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt8VPWd99+TK7kykxtJQIebqZfaqogtrhKNa1ooEKl9\nCrpVn9ZNW6x9VWsbCi7aVUt01NZ9tq21KVZlFey2YkQNohhj3QWK2liCF5RIBCc3yEwmIZnMTOY8\nf/zmzJwzSSBAYibh+369eIU5c87vXLh8zvdu0TRNQxAEQRCEmCVurC9AEARBEISjI2ItCIIgCDGO\niLUgCIIgxDgi1oIgCIIQ44hYC4IgCEKMI2ItCIIgCDHOiIj16tWrufjii1m8eHF4269//Wvmz5/P\n0qVLWbp0Ka+//vpInEoQBEEQTjksI1Fn/eabb5KWlkZFRQWbN28GlFinpaXx7W9/+6QvUhAEQRBO\nZUbEsr7wwgvJzMwcsF36rQiCIAjCyTOqMesnn3ySsrIybr/9drq6ukbzVIIgCIIwYRk1sb722mt5\n5ZVXqK6uJicnh8rKytE6lSAIgiBMaEZNrLOysrBYLAB885vfZPfu3cc8RtzmgiAIgjCQhJFaKFpo\n29vbyc3NBeDll1+mqKjomGtYLBba2yeuuzw3N0Pubxwzke9vIt8byP2Nd06F+zsWIyLWt912Gzt3\n7sTtdnPZZZfxwx/+kJ07d/Lee+8RFxfH1KlTueuuu0biVIIgCIJwyjEiYv3ggw8O2Hb11VePxNKC\nIAiCcMojHcwEQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBOHXo6HCzcmUt\nTU2Z2O2dOBwl2GzWsb6smEfEWhAEQfjMWLmylurq6wAL9fUasJ6qqqVjfVkxj7jBBUEQhM+MpqZM\nwBL6ZAl9Fo6FiLUgCILwmWG3dwJa6JOG3e4Zy8sZN4gbXBAEQfjMcDhKgPWhmLUHh+Pysb6kcYGI\ntSAIgvCZYbNZJUZ9AogbXBAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFr\nQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhx\nRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBODE6OtysXFmL02mjsLADh6MEm806rGOamjKx2zuHdYww\n9oyIWK9evZrXXnuN7OxsNm/eDEBnZye33norn376KdOmTeOhhx4iIyNjJE4nCIIgACtX1lJdfR1g\nATRgPVVVS037RIuzz9dDTc33AQv19YMfI8QeI+IG//rXv866detM237/+98zb948XnrpJb70pS/x\nyCOPjMSpBEEQhBBNTZkooQawhD6b0QW9vv4qqquvZ/v27mMeI8QeIyLWF154IZmZ5j/wbdu2sXSp\neltbunQpr7zyykicShAEQQhht3eiLGoADbvdM2CfaEGH7GMeI8Qeoxaz7ujoICcnB4Dc3FxcLtdo\nnUoQBOGUxOEoAdaHYtYuHI7LAbPru61tD1AM2AAXkyY5sVr/CBxi3rwMHI5FY3cDwrCJuQSz3NyJ\nHdeW+xvfTOT7m8j3BhPz/uLi+klOTgQgOTmBnJwMsrIyuPnm5w2x7DKmTbuPgoJzaG7ew8GDt6PH\nuDMyNpKdncFNNz3Pxx+nM2NGFw8/vJCsrNhLOJuIf37Hw6iJdXZ2NocOHSInJ4f29naysrKGdVx7\ne9doXdKYk5ubIfc3jpnI9zeR7w0m7v2Vlz8XFuVduzT6+lSy2N69KRhd3zk5Z/LCC5dRWtrPwYOR\n7Xv3pnDjjYOvEUtM1D8/neG8iIxYnbWmaabPJSUlPPPMMwBs2rSJK664YqROJQiCIDB0gtlQsezB\ntg8nSU0Ye0bEsr7tttvYuXMnbrebyy67jB/+8Id897vf5Uc/+hF/+ctfKCws5D/+4z9G4lSCIAhC\nCLu9M1R+pdzauijrsWxVruUJx7JXrZrDrl2VuFzTsNkOsnr1EtaufWvQNYTYYkTE+sEHHxx0+2OP\nPTYSywuCIAiDMFSCmc1mHdSVXVn5Nk7nKsBCb6/G2rXrhxR2IbaIuQQzQRAEYXjoojxYTHewTmWD\nubyHEnYhthCxFgRBmIAYu5vpncrsdk1c3uMUEWtBEIQYYai+3SfSz3swK/rpp+cgLu/xiYi1IAhC\njDCYNVxVtXTI7UdjsOQzcXmPX0SsBUEQYoShyqhOpLxKEscmFiLWgiAIMcJQpVjm7S7a2t6ltJSw\nS3ywphojYUXLOM3YQcRaEAQhRhjKGjZub2t7F6dzFU6nconX1T1AaelU7r770mEL6XBF+ETc78Lo\nIGItCIIQIwxlDRu3l5aC0xlxibvdZ/KnPy06rjahwxVh6W4WO4xYu1FBEARh9IluGQpqPvXxCOlw\nRXg4IziFzwaxrAVBEMYRuku8ttaPx5MCLAQ0CgoODXuNoWLjQ51LktTGHhFrQRCEcYTuEr/hhv+i\npiYBeBY4xNtvu3G53ANiz4PFp4crwlLqFTuIWAuCIIxDmpsLgF5gOWChtVWjomJg7Hmo+LSI8PhC\nxFoQBGEcEG0hFxT4qK+fwrFiz5IkNjGQBDNBEITPiI4ON+Xlmygt3UZ5+TO4XO5hf69byPX1V1Fd\nfT0QoLBwN8dKAJMksYmBWNaCIAifEdEu6V27KqmtvS4cZz5aSVW0hdzcXEBt7SIqKgaOyDQiSWIT\nAxFrQRCEz4howXU6P09FRe2Qgmx0WRcUNFNf/xSQAXgoKPAcdUSmznCTxKRbWWwjbnBBEIRBOJbL\n+kQwu6RdwLts3Up4/aO7rBOBa4DFwLWhzyNHtJu9oqJ2RNcXTg6xrAVBEAZhNFptOhwl7NpVidP5\neeBdYCW9vRaqq9X6DkcJfX3r2LEjDjiMz5cWLsdqbs7B7AbPOalriUYS0WIbsawFQRAG4XjFaziW\nuM1mpbb2OsrK3KSkFA5Y32azkpychNv9bdzun1JTsyJs4UZb3QUFLeHzLVv21Elb/pKIFtuIZS0I\ngjAIw+3ypROxxDupr3+RurqXKS6OHxD71WPI5eXPhCxq8/pDvSREJ4r5fAkmy/94eoMb0WPV+/Yl\nUFhYSXZ2ETNn9kgiWowhYi0IgjAIx5tFHRHZGmABbvcWqqvT2LXrCWprrx+QrOVwlODzPcL27V1A\nNj5ffzhuPdhLQnSiWGnpNoZr+R8teczo7geNuXNlslYsImItCIIwCMfbajMisunAFvTOYk7n4kE7\ni9lsVpKSUnG7vwdYqKnRSEpaf9SXBKPotrXtAcoYjuV/PCVhEquOTUSsBUEQBmEoa3So7brI1tW1\n4HafyXAEcDChPNpLgtkKLqawsJK8vLMpKurl7ruHtvyPJsjH6+4XxgYRa0EQhEEYyhodarsusi6X\nm8svfwKnczFDCaAu+Pv3t6CSuoYnlGbRtZGXdzZbt15Bbm4GH3xwgPLyTYO6uo8myNI0ZXwgYi0I\nQswylo06IsLoBmrC9dD79iUQbaV2dLi55ZaXQiVXh5gzJ5kvfnEdzc052O0eVq26wCSkPl8PNTXf\nBzqBDVitXoqLE44plEcT3aO5uo8myDJZa3wgYi0IQswyGrXOwyXSMcwJ3Bauhy4srCTaGl65spYt\nW24Mb9u2bQNlZQG2br0CgPLyTab7sFofCO1rBa5l+vRnqaq6Ilz+NdTLiS66+/bF09HRRGNjEeXl\nz/Doo2VHdXWLII9/RKwFQYhZxjb5Se8Y9rzpGrKzi5g7V1mpBQUt+HwJvPZaErABWIgS4AyamvrD\nK0XfB2SjOphtAdJoa9uDyzXnmC8nuuhef/3TNDSswum0sHu3xo03PoHdjsSeJzAi1oIgxCxjmfwU\n6RjWhdGSnjmzJyygRotZ7fMgUAD00NbWTmkphnGWkTXmzQvyzjsP43SuwpgxPtyXE+Vuj+xXV6ex\nY8cVSOx54iJiLQhCzDKWyU+RF4WFDBVXHmgxfw5YRHLy7TidP8XptFFfr7Fgwe8oKzPex1dYtuwt\nnM7IsVu3gs023HKsQxhfIOCQuLonOCLWgiDELCMtQEdLWIv+bvXqOUReFAI4HFcOSG6LtvyhO/T7\n01Eu7nSgiwMHMnn11SVHPba3N5He3psoLKwkK6uIjo697Ntnp7z8mQGx63nz0qmp2YCawNXF/Pky\nHWuiI2ItCMIpw2Ax4fvuu5yVK2upqwvgdicDl1FfP5nBktmGEvTaWj8eTwrKCteAg4BqdgIaHR2V\nA65F9xps3Qq9vX7Uf8dv0NOTwFlnfUJDw3SczgwaGjz4fM/z+OPfCl8DJGG1eoGDzJuXwaOPXkN/\n/4BTCBMIEWtBEE4ZBosJR7fbhI3ANYPGi4dKALvhhv+ipkYD/gDk4PMlocqyAGpwuQoHWMjmHuEp\nqGQ2C273Il5//U7g1vA1bd/+AKCEuqRkfTjWDarrWVaWdch51sLEQMRaEIRThsES1gbGndOJjhfr\nFvXWrRj27WTz5oMUFf03gcCh0PbbAQuapqGywy3ActMYzGhr3eEooa7uZdzuyDX090+PuqZsQL0s\nqPGa0h70VEPEWhCEU4bBEtYqKl41CbjV+j7FxS5TIlnEot5AJLHrRYLBVSGR1YDHMQusDzWF+OjC\narNZ+fKX+9myJXINOTkHaWszZ4+D7hno5ni6ngkTAxFrQRBOGWw2azhG3dSUSUXFq1GJZB4cjuUD\nEsn27YtHucctwL1YLFPQNLMQQzvmDG0L8Klp2/vvv0lJyRFmzQqYXOIWSwD1IqASxs49N430dHP2\nOOiegSWha0mjsLABh+O6UXteQuwgYi0IQkwQnby1atUcKivfHvFWoyfSFa2jowmIxImTk9fg9Z6F\nWZyt6CIKO1BW9W3AfcDZwBG83ttoaNhCQ8P1pvM2NxcAV4XPd/jws2zYcMWA61Cegc2hZ+LG4bgO\nTYNlyzawd2/KZ96SVfjsELEWBCEmiBbRXbsqw4lUx9NqdLDyrNzcjPD3J9IVbfLkqTidG9FLsU47\nrZDZsz1s3/4A3d2ZBAKTQmumAW8CPwXeAGzAOcBiw2rpA8473OYvg5WyRbcy/SxbsgqfHSLWgiDE\nBNEi6nJN40QSqQaznJ999vrw98cSxsHEvrPzU4yW9ZEjlfzqV9excmUtjY2pHD78AR5PBt3de1Ad\nzGxEOp8ZO6C5gJ1AO3v2OLnhBicPPbT4pJq/yDzqUwMRa0EQYoJoEbXZDtLbG/mcn38oPOSioKAZ\nSAxNtTK7fo8lXqtWzWHnzntoa8sjPv4Q3d3puFzu8PGDiX12dpGp21h2dtGAkq/k5DXARcAelCir\nzmeZmR34fHfg9Z4P7AXuBiz4/VqosckLJCWlnrC7X+ZRnxqIWAuCMGocz4jLaOty9eolrF0b+ezz\n+amuVpOt1DSsaxisucn+/QHgSeBrwOQB4lVZ+TYtLbOAawgGLWzbplFREXEdDyb2M2d2snu3uT94\n9H59fRcBS1Au7/tISSmktBQcjjKWLXuL+vqrgM2mYyCD7ds/xe3+HsNxYw/2PB2OEpKTN4Zi1tIT\nfKIiYi0IwqhxPMlcg8Vjq6rs4d+Xlm4jInQZGEVv61bYtesJnM6bUC5oNYayuHjKAPFSmd1O1DSt\nLmAhdXUBGhubqKx8m/37W4gujVq1ag67dlXick3DZjvA6tVl3HnndswJZkfC1wNTuOyyI1RVqa5j\nEeu3K+qYLlQN9fDc2EM9z6efvkaaokxwRKwFQRg1oq3PfftSB8xr1jSGZX2b3b0ejKKn+mqvRu8+\nBhamTz+DqqqBGdXRmd2wAbd7Epde+if8/n9HdR4zD+6oqKgNJ7v19mosXVpJd7debuUDmoHvh86g\nAclApP+ncQ71oUP30NMzlbi4w8yblw4khLqfHduNLfHpUxcRa0EQRo3oeGpHx14aGswZ3sCg1uJQ\nfbhVL20f8ERo3URgAZFsbDia6EXHn5XYXoXfHwh9tqLizVU0NZ1BRcWrNDamYRTJSBexxSjX9lVA\nDSrT+wPgX2lufi18zqMNJHG53CQlDS+5TOLTpy4i1oIgjBrRceh9++wGoeykrq6Vvr4pKAt1IWAN\nW4tDuXxVL20Vu1ax6eXo4lVY2EBeXvCoojdz5pFQ/LkTeDG09QXgI4zdydzun1Bfr85dWLiWgS5v\njYgrezLqheFFIAd4gYKCwYV0sLjzcEutxnJkqDC2iFgLgjBqRFuU5eXP0NBgFkTzAI3lYWtxKJev\nUbCUIK4LZYV7cDiuO2YmtX58bW0LHs9PDedfh8redtHb68bvj8S0s7KmM3euOmdb27s4nStQrvh7\ngQySklbS359Mf/9cVDvQBcBfBj3/8cTxT0bYhYmFiLUgCJ8ZRqHdv99rGl6RkuKntHR92FocyuUb\n/QJgFLSKilePWfqkH19auo36eqM73A/00tX1KZr2C4wx7Vmz+sOu+VtvPURPzyaOHPkYv//HgA2f\nL5Kdrr94NDfnDCq2xxN3PpFua8LERMRaEITPDKPQKnd2RIxLSzEJ0VAu32gB7Orq5NVXf4guaD7f\nOh5/fNmAc+vH7dsXT0dHE93d8Zhd25OBa9G05zCKqdXqxeG4ElDiWVNzI2ZvwDVEZ6dDGna7e1Cx\ntdu1YcedJaFM0BGxFgRhTDhW/HWopKxoAUxMXItR0LZvjxtwzOHDxjnQG1HZ4CrrOzPTS3d3C8Hg\nTaG9zVOtiosThmy4EkloM2enT5q0i9Wrl/G9731EtNg+/XT04BBJKBOOjYi1IAhjwtEypI9GtGD2\n9+dgtpAPDzjmpptqQhncnahJWJF4dFzcM+Tnazidk0N7LwDuwGqdQXFxAqtWXRAuN2tr2wMUo2q5\nXUyatAuLxU1m5l407S7a2s5HDez4MZdc8is0bSrwGCpbXDVoOZ77loQyQUfEWhCEMWG43c2i9yso\n8Jmszby8Vlpa9PGSrfT2urDbN2GzHWDTpjJmzLDz8cdqAIfK1r4NYzwaDvPHP17OkiVr6OubgcXy\nMZdcks4f/nAlmgaXXfY4LS0/ALYA55KYuJaUlCx6erLwes8EvkZv72Ss1gdQHcwUfv+Foc9DN2g5\nFif6QiNMPESsBUEYE4abPBW934IFv+OKKx6hrs5CMHiY/v4errjiEIcPp/L++014vSo5rLfXRXHx\nL5k9+4vs21cPfA7owezG9jFvXjq//e1H9PWpnt2appGVtR5Ng5KS9bS0fAEl1KpEzO/vxu83J5Op\nuHU2Q3U0G6pBiyAMFxFrQRDGhOEmT0Xv19xcQFvbuwQCqrlKe7vGe+9VUl9/RSimq++7Ba/3Lhoa\nLMDVKFFNxyioU6Y0AqezdStE13qvXFkbcp13o4+1VEQnk6k1580LkpS0nrq6AG53K8aOZhJrFk6W\nURfrkpIS0tPTiYuLIyEhgT//+c+jfUpBEMaI4xncEZ08ZZyqZTx2sCSrDz4wj89U4zTBZjsQmtTV\nCfQxUFQvJTPzfk4/fSYdHXvp7k4JZXfrDVKeBRIpKPDQ1FRApGb6d6huZQNbnVqt71Nc7MLh+Ao2\nmxWXy80ttzzP9u1/ALKZNy+Iw/GVkXvIwinJqIu1xWJh/fr1TJ48+dg7C4IwrhnKtR0t4qtWzaG7\n20Ni4lr6+3PIzm7i7bcTaWubA3RTX78E2ExV1VJWrDiDmprb8fnsQCtvvHGE9HSLaXympn1ISclL\n+P3dJCSsIRBIAaZjdkt3A5NJTw8wa1ZPqO3p86HvazDXSa8LvSQsAZ4DJmOxrCEjYzpz5/aQlGRs\nxLLc9EJis1l5/PFvfRaPWziFGHWx1jSNYDA42qcRBCEGGMq1HS3iu3ZV4nTmoeK8GbS3dwA/wxgH\nbmrKZN++JhYufJ5gMNKk5PDhDcTH/4NJk9agaTPw+/fh9f6UhgYbEXd3MlBCxPX9D1QG94N0d/tp\nbEwNradPwUrH7GrPCZVYbWbfvgQ6OtxkZ5/HzJlHcDiWhsW5o8NNRcXwPAmCcDJ8Jpb1jTfeiMVi\nYdmyZXzzm98c7VMKgjBGDFUXHC3iym3dBhgbjAxsKnL11c8RDH4u6rsM+vtn0t9fTmFhJU7nl1FC\nrH/vB3YDS1HWsga8AawGLHg8GocP672+F6Ji1fuARabr1jOxy8s30dCwCqfTEuopHvEWqNptFdeu\nr19CX99fSE5OEvEWRpxRF+uNGzeSm5tLR0cH3/72t5k5cyYXXnjhaJ9WEIQxYKi64GgRV7HlwtBn\nN7AntIKKERcWNrBq1RIuvvgg8CEDZ0C3As/jdPaihHmx4ftEYAZKhDOIWM+6ld1FZmYuUGmYnnUd\ncB9wNoWFDTgc14Xv6WjeAn1spr7+jh1xuN3SHlQYeUZdrHNzcwHIysriyiuvZPfu3UcV69zcjNG+\npDFF7m98M5Hv70Tv7fBhNzfdVMPHH6czY0YXjz66hKwsszX56KNlrFixMbRPN2vXfotLLnmMlhYN\nFS+OuMCnTbuPd965iRUraggGVwGfALejYtDtKCs6H7gUaADsqIEaBSj394LQmkYSME7n6u6+j6lT\nz8XpXBzeIzW1kEWLjvDwwzeZrr+oqMf0olFU1EtubgZOp41ob4DF8qlpm9Np+8z+zkzkv5sw8e/v\nWIyqWPf29hIMBklLS6Onp4c33niDm2+++ajHtLd3jeYljSm5uRlyf+OYiXx/J3Nv5eXPhePRu3Zp\n9PUNZk3G8+tfLzJtOf/8PGpqNgD6HGkACzk5Z9LfH8/evSmh7XagAvgD8AVU/PkHRIu8Emz98/6o\n78wtSW222WRkfAw8hbK+D5OW9iF7987lO9+pNrmvf/zjL/DGG5W4XNOw2Q5w221ltLd3UVjYgdHi\nLyxs4ItftFJTY9zm+kz+zkzkv5twatzfsRhVsT506BA333wzFouF/v5+Fi9ezCWXXDKapxQEYYQY\nbhnWiQ6baG4uQLXhfAyj6L3zzm7OO28PZ51lrImeDEwFFpGfX09Ly2Sik8JU0xPlyk5IsBIIGL/L\nMp1j5swedu7sBH4Y3tbevoH29qvCCXB5eWdjt3fi8/nD7u7eXo21a9dTVWUfxOWvXOdJSdIeVBh5\nRlWsTzvtNKqrq0fzFIIgjBLD7TA2VFLZYOValZVvG9qGHgkd14MxvqxpBTidNxIM3kNZ2XoaG1Np\nb3+Pnh6NuLg/cs45mZx//jq2b+/A7Y4khcFeVCMSK3APA/uFb8Bq9VJcnIDDcTnnnVdLdOKa/nun\n8/M4nUuor9ewWv/IYC8jQ7UClRi1MBpIBzNBEAZluBbzUEllt9zyElu2qGzv+nqNF164g0DgrvDn\nBQvWsWDBOmpqfIbzgJpkZaGz024Yp9kTfnHYts1Fbu6DdHUB3IPFkk1c3Kf09/8EJdQagUAPkYSy\nbiwWK0uWBHA4rgx7B1SSmwvVSjQNleR2KcqKj7QKhUMYhV+6kQljwcBZcoIgCCiLWYkUgMb+/R9S\nXv4MLpfbtJ/NZuW++y7Hbvewb18ql1/+BCUlz7FtWwuqMxiAhUDAjlH8X3stgaSkROLjm4GvEmnr\n+S7gQtP2hs9lfnF4hvb2NPr7LwJmoWnX0N9fBNRgtT5KYWElKhltOSpLfDmTJ3cDsGzZW+F72LSp\njEmTfhnabwnwMzIz/5NJk+5AJao9BbiYNy+DsrL1nHfes5SVrT8h13ZHh5vy8k2Ulm4b9BkKwrEQ\ny1oQhEFxOEro61vHK68ECAS6cLu7qK6eyvbtf+Svf/22KX5tdJmDhtO5EZXBvQG4FiX6jRgt1N7e\nZKqrl2Ox3INxUIYS2Pvwem+jokJ1MTO72l1EN1BRMenFTJv2JJ98YkH913Ynykp2091dSHV1PHBZ\nKCb9MHl5ZzNp0gy83sgLRFzcJLzefwuvXVhYyUMPXXfStdLDDSkIwlCIWAuCMCg2m5Xk5CQCAWPj\nko20ta3hllseISkpNRx/bmxMY2AfbjXVCjaj6qKTgcdR86SnAN9ATbmaiu76jhxfAPyOLVuslJc/\nw+rVc9Bd7Q0NGVHJY16U21qjo6MJj2cFSvwvBHYBd4X214UdnE7V5ASexBzbzjZdR17e2SPS1ORE\nk/AEQUfEWhCEIYkWGV2Et2/vwu3+HrqlOGXKHShhzkANuuhFiV8Lqq92I5oWaRmqLG5QrmY/8L+Y\nG5skAT+jr+9RqqsTqav7G8XF8Tz99Bxuuul5tm0zCmwLaWk9/PM/r6exsQin02ilw8Dr7yISz+4h\nM/NeZs48C7vdg8/Xbyq9Gqn49FBJeIIwXESsBeEURs/YdjptFBZ2DCjPihYZFVceaIEePpyAcRBG\nQsJdBAIbUNnZk5k82YXbbRRNN/BrlKtcubaTk9fg988kGExBNTbRXd7fwe22UF2t3MdJSQA/B+ag\nLOrvEx9fFWoN+gy7dycSEWM9acyGPiHL6+3E6707fK3p6ZVs3apmTbtc7lEpvRoqCU8QhouItSCc\nwkTHmqNjqQ5HCUeOPMJrr2kEAk7i4rK45JKH2Lv3CGoaVTdwMYFAPkbxPuusc5g5s4emptcGtVhV\n1na84RgbfX3TgY+BuahxlQuAHAa6jzNRLvUl4evs6ckIX++WLY/Q16eL8SLgDmy2WcyfH4fDsZxv\nfGMnu3dH1szOLgqvM1Q51skyWusKpw4i1oJwCjBUg5NjxVJtNitPPfUvpm3l5ZtoabkFXXgtltvR\ntHOIxH5d7NnzNh99VITNtodHHilD0+CVV+7E75+NillfS2Lievx+o4A3Ar8wrZuRkYHHE+0+1qiv\nbzadr7+8jtPXAAAgAElEQVT/AKWl27DbO5k58wzee89oxV/A7NkJVFVdBsDMmUdCAzkiDVIEIdYR\nsRaEcc5wOo0NlY18tFjqcAVe07JQFnEVqnd3F8FgJb29quPX0qWVzJ07Db//34kI8wbmz8/kf/7n\nDrzeuSh39udN606ePJudO6/kllseYfv2LiAbn6+fn/98Hps3dxAMRlzdmvYL6uvVveXn/wJz0th7\nfPRRIeXlz+BwlIhLWhiXiFgLwjgnWojr6h6guDjPJNpDWdC6cKmYtcskXMMVeNU0pNLw+bemc7lc\nBQPOHxfnYc8eNxbLLJQrfSHK9R1Zd9IkJwBJSanhZLaaGo2kpPV85Svp1NQsN5wzsnZPTz6RjmgN\nwApcLls45l1VtXTYLunhtlwVhNFGxFoQxjnRQuh2n0l19SKM8eehLGg9ljrYoITIum6ghq1bCZdR\n9fWtY8eOOI4c2Y/ff6bp/Kq1Z+RcmtaI3T47dP5O4EWCwSAtLbOAr6FqoTcCC4iLu51g8MvAEVpa\nfkBFxeZBXzSefnoO77xTidM5DeVWj2SS9/a2ohLXQL0IbEHPAt+3L/64BFjqo4VYQcRaEMY5g2ds\nm+PPJ+L6LShopr7+KVRJViK9vUuorp7MSy+t4Z/+yYbb/R3gEeAAZrezF2OrT6/XRm1tVyi2nQXc\nZth3I3ANKSl9XHbZ0/zP/2Tg8fhQXczcvPiik/nzC03rt7Q0cMstzbhc01D/hX0/tE4asAO/f7ph\n//0YG6h89NEdfPnLTtzunzAcAZb6aCFWELEWhHGOLsR1dQHc7kkol7Kyns1WpMbTT885DjduIsZy\nLF1Yvd6LeP31N1EW60qUtfwEaiBHO6qLsdFFvQGP59rQPtEzoPuA5wgEGnj11UmGLO6rgY34/d+n\noeEOpky5k9bWTCCHlpZp1NT0oCzq7xPp7f0ukAt8E3gUVfaVazqf13s+Xm8iwxVgqY8WYgURa0EY\n5+iubJfLTUVFbbhcyuG4nIqKod24RiEvKurh7rsvNQl5c7O5bEpZyhpwhP5+O5GuY1ZUE5PridRG\n3wecg5o9nQc0AR+iyq6Mk7KSgCX4/Xpf8IENWDyeWSQnt2O2yO8AZgPVqBeEbuAW8vP/MzQ+MxX4\nDip2bbT6+1CW//AEWJLRhFhBxFoQJgjRtbwdHW7q6gIMZUVGx2O3blWJafooy/37WzAL3QcoUfwq\nmnY/0EYkVmxsF2pDCfWi0P7LUeJ9F8oK3xD6eRi4OXSMGo9pPp9qwKJp+4AZmIXc+HKgERdXyeLF\nm1m9+uusXbuerVuht9eC8jJsJDXVj9V6EKdzRegY87jM4T5TQRgrRKwFYYKycmUtbncyRgFsa3sX\nl2vOoCVYbvdsqqt7eeGFZwkEfoBqevI4CQkHiY930dcHyjLdiKaBGtDxBMpSNSd5KYu6m0gnsjwi\nVvi1obUnh36BalGqhFUJ//+GjrkTTZvOpEmfmu7DYslG0yLXnpmZHxbVqio75eXPhLK/rcByFi3a\nyN13XxdOWLPbzeMyBSHWEbEWhAmKEuPLiCR7fYDTuWKISVYaKuY7hUAgDlV+dROwhUDgCwQC/wvM\nAv7VsP8TKAs3A9iHxXIP8fF5BAKTQts04K+AhylTPqa11Xiut4F+LJZ7yMgoJDHxQw4f/hg4HdUi\nVG9peit9fRZaWlwUFlaSl3c2druH7m6LqT/4vHlB071Hu68ffngJ/f3xYiUL4xYRa0GYYOix6P37\nA8ALRMqjGgAL+/bFU16+iX37EigsrKS7Ow+PJxXoR8V6VwHPM3Bs5YOYXdFeIq7opSQn34HFkkIg\ncD1qulYkOe3ccx8hGFxDe7sVSEFlmF+IpnnxeBaQmPhbIuVWoCzvFoyu9by8s009vCsqjLHkr5ie\nQbT7OitrYGmaIIwnRKwFYYKgi3RdXWu4NElZqA8CU1GZ0y/S0dFEQ8Oq8PcLFqwjI8PCn/6Ui7KI\nLaj4cXTCVzbmmHIbcC9wJtCL16u3En0KJeSRY1991UJcXAoqSWwjymqPZJn7/YVRax9BxbUHTwST\nWLJwqiFiLQgThEjC2POYRfZzKMsYrFYv2dlFoVnO6vvm5hxefPEqtmypxOPJRAnkQuBhzHHoD4B7\nUOVQn6Cs9dNQ/43obvR7UWIcB/wB1VAlm2CwjWBwNsYs78j1pQE+EhPvxO+/ECXUXwX+jB7DLixs\nwOG4bljPYbCmJ7m5GcN/kIIQg4hYC8I4xihMjY3NKGs0Oqv6g9C2i0lNbaGpqdXwvYuWlgYuuiie\n1FQfHs8hIsM0koG1wNmo+dR+4N8M696DuQ57PxExVo1U4EbD9/eGfkZf35vArcyf/0fee68Bl6uA\nYPBBEhPzSEhwMW9eBg89dJ0pGexoXcgG6zr27LPXj+RjF4TPHBFrQRjHDBxxuQFlFW/AYulE0yaj\nksImAz/B6ZyDsnZ19/X7tLTcTkuLGiepYtjxeDyRrl9qNvVUVDmWbhF3hn4+jxLfhahY9FMoV/jn\nQvsaLegzQ9fXgYpPzwQ+4owzTufsszfj82XidN4aPm9f30ZgOe+8U3nU+46uH5euY8JEJG6sL0AQ\nhBMnWpiURfssYCEpCVSZlJVIzPkaVLz4Z6i4snnSVV7e2cTFTQltawLuIxA4DTW+8gPUCwGooRv/\nhnKTXwO8SE5OV+j35aiMbo9hfw34O6rL2b+EzvuvQCVnn51OVdXSIZqwdOJ0JvGlL71MefkzuFzu\nQe/bKMh2ux7rVueVrmPCREAsa0EYx+Tnt2N2KX+KsliXk529FqfT+F02A8XQQ3QS1/79h4hY6SsN\nx98R+mVHZY4b1+rC69Vbieq11A+jeocfRjVKuZWMjN/j9f4Wv/8H4WN1oR28x/mLwG243Zbw1Kyf\n/ewC3n//TZStoWq5jYIsXceEiYiItSCMA4aK0VosAZRL+xxUYtZNJCb+ioUL13PTTZdTVnYHXu8Z\nKBHXk8f0lqC7gHxgLSkpUykuDuDz+QkGe1FCrTcyIfTzDOA6VH11E+aXhAz6+oyNS14GvoDKLs9A\nxbxtBAIF5OYewOmcjHLHv0hjo5fzzvt/ZGbmUlhYSUdHPl7vx8BZKE+B2YK++urn8HrvDp970qQ7\ncDi+G35WkikuTERErAVhHDBUjLa5uQBlyR5BWco1zJ49i6qqpZSXb8LrvQtdnJOSKklIuJ2enjhg\nGiqurGqws7PvIzm5kOrqG9HHWMI+zILsDH13AGVdr0G9JAAsJCnpMIHAGjStCJVsdhvKotbLxzR6\nexPp7b2JwsJKenoScbt/gsdjwePRcDo3AuUUFq7F6bwRlQneHzpeXdP+/V48Hv2zcu9bLGdIJzJh\nwiNiLQjjgH374ol0IusKfdZdx06MYyA7OytDx6QSsUq34PPdh893H2bX9qNAKocPF1BX10JEBK8F\nqlCZ4fkogc5CDc643XD8htC+GoFAK5p2t+E7NaVLfc5An1kNVvLyzgagvn7g4I7U1Azi4n5PMHgm\nyoJ/mMREF37/atzugee12Q6OwBMWhNhGxFoQxgEdHU2ozmJKrDo6lCA7HCVs2/Ys3d0RIXc6M7nh\nho20tzehRk0aB20UYnZtu4Dv0NtrobdXL69agcoeTw/9Mo67/H3U8T5SUp6gtBS2bp0V9V1a6Pca\neXlO2toy0NuPFhR4SEpKHSRGrXHwYDvB4C8M2+8jIeE0/P7I2gkJXSQmPoHNdpBNm5aMwBMWhNhG\nxFoQxgHRjUy6u/MpLd2G3d5JWlob3d03YxS3mpofkJFRScQa34PK3Nbjyrqr20qk3MsKTCUh4X7i\n4vz4fBehksOMAtwedbwPTfuE1auXs2tXdUjw9VjyLs48Mxjq5Z3Ntm3Gmux14USwxsZUDh/eS1aW\nnVmz1g8i+oXYbAdMa3/taykSlxZOKUSsBWEcMHPmEXbvjoiVxzOJ+vqrqK/XyMy8n4Edyyx0dWWh\nOoFtQcWoVwE5KDd2GrAas8t6OZBIIHAPSsD/GZXR/RyRCVr5qASzT9AbpHi9LoqLf8mMGbPp6FiD\nxTILm62ZTZuWMWOGHYDS0m2ma2xuzglN7oL4+ATmzp2KwzEfm83Keef9P5Mwx8W9z2OPLeI3v5EM\nb+HURcRaEGIUYwZ4QcERFixYR3NzDvv3f4jbXR7ay0JcXA4DO5Y9h7Ki70fVNH+CyuZuAaaj/ukb\nBb6XSExZjzHXYIyFq45lFtRkrFzD8Vvweu/ivffUfmVl66mq+qHpXqLLsvLzD1FSsh6n8/NAN/X1\nSwA1DWzTpjKKi+/A650LHCEY/Cm/+c1msaSFUxoRa0GIUaIzwBcseCRUB52NcZqWGg+5jr/+tZfu\n7k+Bi1CW8OmoxiMbMVvRG1ATuIwCvxeoNHzuIjLUg9DPPOC7od8/aTg+zbSfPtXLWGbmcJTQ17eO\nHTvigMP8/e+dtLYas8U3huutZ8ywc+aZc0ICrpAuZMKpjoi1IMQI0bXU+/aZrd/t27twu7+HLqiZ\nmfeTnh7gwAE7s2YFuPRSqKkxCq4+0jJ6cEYGkEpy8hr6+opQJVnXAveRklLIxRd3smePO9yCNLJe\nu2Gdr6EsbTvwIcaBH8apXvX1Gjt33oPXO4nu7kwCgRRUh7PJRJLZrEAaBQVt4WcRbYlLFzLhVEfE\nWhDGEKNAt7Xtwem8CbBRX69RWFiJ2fo1dyCLi8vB6VyK07mFhgYbCQnNmEVZH2kZPTiji4yMOI4c\nyUEN2zgHZWmfTmlpAJhMS8vNqCSyDajGJMmobmcuVAw8DeU670MN67gPOJ1Jk97D5TIniLW0nAbc\nYDi/XtJ1DsrVvhyVABeplT6RLmRHG+4hCOMdEWtBGEPMgzjK0OueIZ3u7ngWLPgdzc0F2O0efL5+\namoiohsMtqISwFTcNxBIxizKHwLrAB8JCXeQmmqnt7cVvz+Prq6bQsdGyrL0TmDLlr2FuW3oY6hE\ntXZUDFwvq1qMst5/HfrcH2rCsiHqOj7GPPAjncjMaj9KvFfQ3Pxa+LmcSBeyW255iS1b1JSv+noN\nn28djz++7LjWEIRYRcRaEMaQgYM4VN0zWPB4FvHOO5U888ylVFa+zYEDqRQWVpKdXcTMmT3s2NGD\nx6N3KFPlUCrTuwg4hEokawb6uPLKqTz++DJKS7dRX38VqtXnFNO5LZaZVFS8SkGBL6r+OQnV4/t7\nqKYo0Znnt6EEWo9xL0QJsB/1wvBjIrHpDSi3ezfqBaAGZWWfvKtbxcONYQOZUyRMHESsBWEM0F22\n+/cHUMlaKlksISGDQCAiOE7n5/n615/D6VyFXtvc3d3K4cOddHbOwFwjnQccxOxyvg/4N/7+919Q\nWrqNtrY9QDHKlW22xHt7J1Fd/VVycx8gLq6SYPBclKguBJ4hMfGXTJ6sceiQUcg7iMTBdXe7FWWx\nbwQuQAm1up+MjB4uuSSN5uYUCgr+Avhpbn52hMqx9AEk+rUdPsn1BCF2ELEWhDEgeg611foAxcVT\n8PniTK5u2EN7+ySU8H0K3IbHsxGP5ybMMWA97mu2llUi1ye0tEyipUUD4oiLewzoIRj8FuamKWnA\nb2lvn40S/UuIWMQp+P134fGsJGJFd6GsZz0uvjD0nQc129ofWidyPxkZbTz00HWjEkueNy+dmprI\ntc2blz7i5xCEsULEWhDGgGj39/TpZ1BVdQUul5va2kiNMXyf/v77gVtRcd9OlGgbY8CHgZ8Dp6Hi\nwy4iItsM/BfG0q1g8FGUqNeiXNyXoBLMsgFzJzRlraeg11/7fJ9HxbHdKBd2H6rZyixUL3Er8+f3\n8NFHHQZvQCRJzelcQUXF6NRMP/TQYpKSamlq6sduD+BwLBrxcwjCWCFiLQhjwGClSR0dbm699QW8\n3lTgXVQf799hsVhD+3Whz3c210w7iSR96f29P48S4FuBNxgqLq72vxPlEg9gdqufTWLiLvx+Y1xc\nb1eqZ3FvBCJWfn7+PVRV/V+WLXsr1B5VT1J7At3CHq2aaRmNKUxkRKwF4SQYrFxI0zhmCdGqVXPY\ntasSl2saNtsBVq8uY+XKWmpqMlGCGBHI/v5VKKH7J1Ss2Si8XSir1rgtK7R/AcrCPow5lpsVtX8i\ng7ce3cP8+Vbee68y1GnsCPA14uJuJxiczWA13IcO5bFs2VuG2Lhu4SeG1tyA3R44mUcuCKckItaC\ncBJEdxnbseNOLJZEWlq+SHQbTSOVlW+H3MRq2tXatetDFmc8oAshqCztIpYsWU9dXStudwCz8LpQ\n7m/jtkmoGHRC6LNuMetx5vej9j8Ns3gfAdYQF5cNTGLTpq+wdu3bNDVl8v77/43X+wsi5VnmGu5A\noIv6+u8BZRQWqpeR3t5EdDe61erF4bhyJB69IJxSiFgLwkkQHXtubc3E7KbeyL598dxww5Ns394F\nZDNvXj8HD9pMx+lWeH19AsqtHRHA5OSPqaqqoKTkJdzuuURiyZ+gBnTEoVzZXyAu7m0SE0/Dau0h\nP9/PO++sQpVyAVyKcksfQrnN1QuFwije7cDdBIMWtm1TLxL6y4YqrzKWZx1Cud3PRDVJ0T0IFvLy\nzmbu3E6qqyO13MXFCdKoRBBOABFrQTgJomPPaqqV0UpN49Chf9DQMBNVp2yhpkajsHAtRoFsa3uX\nRx5ZwvPPr6e/H2ANMAP4kOeeUz2yOzo+QM2n/hmq3KsIVaOs1igsrKS2dkVYDE8/3YHRnR5xbx9C\nJY3prURdpKTcyec+d0FoSMh0ol8kdDIzP6S39ymUlR4EWiksTCMvz0Jb236czhWhPbVQOdbxdyIT\nBGEgItaCcBI4HCXs2mWM6YJRhAsLG+juzid6KEZW1nSCwXtCrTgP4XTm8POf/5WMjM/hdn8ntJ+b\nxMTf8KMfHaCx8QX6+rJRTU+CqCEdlqg1i/jRj14KNQc5hNc7jYHu7TtQGdr5JCSsITGxCJvtIK+/\nfiOZmVmUl3dSXR003YOxWcm5506ltdU8lzovL4etW6/A5ZpDRcVmkzBL0pcgjAwi1oJwEthsVmpr\nr6OiQh9l6QbWceCAlY6OvWRl2Wlvfx9lyUYEsKOjiba2eIwNTF555U5SUuyoEqgkQMPvn857730F\n+CbKMs4Grg8d86RpzY8+eoeGBqMlvQqze/sQytJ+ArievLxK6uuVkObmZtDe3oXDUUJX1yb++te1\n9PfnkJfXyurVXw/fb0tLtOcgKyzmgwmz9OsWhJFBxFo45TlZQRlMpMrLN9HQsCpUvuQCfoXqo51D\ncvJenM6fAq9hFD6//zz8/q8DT2F0b0cGX6SjMrvNk6+s1qmkprbgdJ6FWUhPJ9L0pBs1IUvPFrdw\n6JCV8877T7KzizjrLB93330pNpuVjAwrfv8PUUM4VMz6vvsms3JlLR988AnGF4BJk/6Ow/HdIZ9N\ndAIerBdLWxBOABFr4ZTnZAVlMLGPTjxLTEwmISEPm+0AaWmz+fBDGwOzsveimo34MIuuPviiG5X8\npR8zGYsFtmy5iNLSF4nUQOvrdaJGUBpFXwM+ADz4fB04nbfjdFrYvVtj69YHKC7OGzCas6kp0/CM\nzE1OZs8+86gvNtHPQeZSC8KJIWItnPKcrKAMJvYFBUeor9cTsRrw+1fj96syrbi421GiOR01ZcuF\nSkzTUIMyEjGKrsXyDzTtDSAXaMNYhlVSMpnKyrfxeH6KEtInAC8WSxslJalYLI/wt78l0Nv7CX7/\n5NCx/4pqQ/p703273WdSXb1owGhOu91jeEZ6k5PNwCJmzVp/1Gcjc6kFYWQQsRZOeU5WUAYT+4IC\nH2ZXduT7YLAIZeUeRDUNKUSJbyJKjL9NxH39Ppr2A5S4bgD+D/AUcXFTyM9vZu3aMr73vY9QQl2D\ncnHXk5CQQnp6DqtWzaGy8m2ami6goaGVQOBaw5V7MFvi3YCFtjYbmZn3Ehc3hXnzgjgcX6Gi4lXT\nM7Ja36e42HXM7G7JBheEkUHEWjjlOVlBGUzsm5qMiVjdmEVxHzAXZSk3oTK0dSt6DZo2GX1spGpu\nYkW5x53AfwM/Ixi04HSqeLLdrlFf/yKq8cgW4Iv4/X+jujqVHTueprVVTzp7LOo6JqFqtnWL/VpU\nYxM3Hk8u8G2SktZjs1kHeUbLhxXXl2xwQRgZRKyFU56TFZTBxN5siS5g0iR9OEcD5vnOD2O0ujVt\nKuaksNND3+k9wfVhHjVAOi+++AlPPTWX6upGlFDrDUgWAxtob3cb1r8KPclNZZtnYh7c8SAwFfg+\najZ2JCQgoisIY8uoi/Xrr7/O2rVr0TSNq6++mu9+d+jMUUGINYzJY0VFPeGM6ejv7HaNp5+eg6ZB\nRUUtH3zQR1LSSgKByWhaFgkJCeTkvMGhQzMwzneO7tsdH3+Q/v7lKOFNA/4GrEe5rI3DPJSL3e9f\nxHXX3YHqIKYnkaWH9rMQF5dNMOgCnkPVWbtRZWT7UF3QjIlsn0OJPOgxdIkxC0JsMKpiHQwGufvu\nu3nsscfIy8vjG9/4BldccQWzZs0azdMKwogRnTzW1xfJFDd/52LXrofp6cnH7U5GtQA9D11Uu7s1\nurvvZaBLvBdju87MzG5crl+i3OTdKGv6d8TFHSYYfCp0nC7cABb6+magyrgexNyx7A4CgWyUq/si\nlBv9bsP390ZdS1doTY3MzGYuv3y9xJgFIUYYVbH+xz/+gd1uZ+rUqQB87WtfY9u2bSLWwrjhaJni\n5u+2hAdzRFzKUzBbrlNR85/10qckoAKYTGbm/Vx+eT61tfmodqLGcqtzCAZ3EElYM2drJyU10tc3\nGbgg6nznh873o9Dn56K+n0JcXCWZmflcfHEQTfPT3PxsyJX/LWleIggxxKiKdWtrKwUFBeHPU6ZM\nYffu3aN5SkE4LvQZ0sYhGw899NWwUB0tU9z8XRpmIcxhYLb1p6h48JbQfsbM7CyqqpYye/bTUesk\nomZbzyYya/paVO/wmUAj55+vMWXKeurqWnC7jefrwzzCMtqqn0QwuBq3WyM9fSO//vWiE3+QgiCM\nKqMq1pqmjebygnDSRGZIR4ZsJCVFXN3G5LGiol7uvjviFjZ+19a2B6fzUpT1qgGfEB9/iJSU/Xi9\nOaSmdjB3bhJJSX/hwAErDQ31GIWzp6cJgNTUZjweo6C+jZqQFT2M42z07O3333+A555bSmNjE5dd\npiey7UG9GNQYzrMAWE1CwukEgx0Egz8I3YmFl1/uw+VyizUtCDHKqIp1fn4+Tqcz/Lm1tZW8vLyj\nHpObmzGalzTmyP3FFk6nMdlL/Xz5Zbj55s08/PBCiopO49lnrx/02I8//pitWz/C651OUpKb5OT7\n6euLCGt//4P097t5//2vMmuWPXzcsmUbaGiwY8z61rQscnMzyM+fTUuLMRu8ELOl3YNyg98U3max\n5JKbm8HNN+8OCfUSYD5KqA9jjImXlc3i2Wf/lWXLnuJPf5ocWkPD5UpizZo3ePrpa07mccY04+3v\n5vEi9zexGVWxPvfcc/nkk0/49NNPyc3N5YUXXuCXv/zlUY9pb+866vfjGX1YwkTls7y/kRoQUVjY\ngbI8jVauxp/+dA11dXdywQWn09ycg93eyaOPltHfHx8+trj4v/F6VUJXX9/AMiz4HL29izjnnDWc\nddaF4evcuzcF1RAl0go0Le1+PvjgAG1tjcBqIpb0PZhd1wdRLvGI0H75ywHa27tC6+qubiuwHKv1\nAdzuVeFrfuGFRzj33Cc57bROMjPvx+M5K3TMQvbufW3C/v2Uf3vjm1Ph/o7FqIp1fHw8a9as4Tvf\n+Q6apvGNb3xDksuEESE6S9vne4SkpNTjFm+Ho4QdO35Pa2ukhSf4AQutrZnU1NwYPseKFea4rsrC\nNoqzLvzmjmB9fRdRX78k3IpUNTHRO5KloHqEZ1BS8gRO57+gLO40EhPfRNM6CQSM1xYAesOJYfPm\nBXnooa8Aegx9Sfj4KVPexGJJQLnmu4EFBAIZNDRYaGhYQWHhWjyeReHrLShoobx8k0zIEoQYZNTr\nrOfPn8/8+fNH+zTCKUZ0lvb27V243SrufDzDOGw2K7m5X6S19RuGrZtRYmseB/nxx+mmYxMT38Pn\n08up9gNWLJZVaNoUVCb4wtA6R8JrNDVl8vTTc/D5nmf79k85csSH378aj8cSilXrE7bgnHOCfPCB\nJ6pF6BPAdSxePPD+VAxdnyftxuc7Pfyyoa7j58A09KSzrKzpzJ0bicd3dSXIhCxBiFGkg5kwLonO\n0lZzno8+jGMo13lHxweYLeJ/oCxRv2n71KkdpvXmzTuNurprUAKryq1UUuUToWP+GlrrJlQzkhfZ\nv99LRcWr3HnnpVRWvs3WreD361neVlRWOeiZ521tB+jtjVxDYuJHLFwYqX8+WjigtHQbZsv/QmAR\nqu5aY9as/rAY5+ZmcP75zx7zGQqCMDaIWAvjkugWnz5fPzU1Rx/GMdQozKys6TidxqQuG5BGUtLf\n8fkiLmhN8wMRgfzb3zKJuLKNomhDJXlp5OfXc/75f2H7dhdu909wuy1UV2vs2lUZVZetZ3nvITPz\nAOnpnTQ2FnHWWekEg/fQ2WnHZjvIpk3fZMaMSLLa0cZ75ucbx2lG3PLJyZlkZ1fS2FhEefkzOBwl\n5OZmyIQsQYhhRKyFcUl0r2qXy01SkhLv/PxD+Hx+Skqeo6OjiezsImbOPGKY0+wGati6FcrLn+G0\n047Q0BA993k+gUAzxlpop3MzYBbIwTuBvQtYsFrfp67u/2KzWSkt3UZ9fUTQW1ryMQu8L3TeFfT2\n/gaPZzVOp1qvrGxod/TRmrZYLAHMDViUWz47243TuSo8xxrW8+yz18uELEGIYUSshQmBUbzLyzdR\nXX0jSvwiohSZ01wDLKe3V1m5Cxaso6xsPXV1AdzuScDngQcJBmcCT6JaeU5mxoxuYKBAKsv7DlQ8\nuAOYDnhITvawbNlb2O2dFBT4TFZrMNiIWeCdwCpAw++fynDd0UezhpubC1DDO9TLSUrKc5SWQmNj\nUagP95kAAB2VSURBVOhFwLy+DOsQhNhFxFqYcETE1Ni9y0J2dhFz565n61bo7Y1s37LFD8SRlLSP\nSy9NYceO9/H7jT221wCn8cYbbXz8cdMQ8fJ/Ae4HvoxyN/8Tra1NtLbGU1+vYbM1oAR9BtCIGk/5\nKOACckhI8HHmmU9y8KATtzsXo5C3tb2Ly6WGhETHp49mDUeuU5VxlZYqC728/JmQRS3ubkEYL4hY\nCxOOiEh1YU4QcwNJJCe3mJK21Pzoa+nr0/jb39bQ3z8Ts+V8EbAEp1Nj6dJKamuvo69vHVu39hMM\ndqFqnp/D3GnsTuDfw59drmYiPb9dwC9DP28DLAQCGtOmraOjw4/bnYkS9rMAC07nCioqlAteud87\nqa9/kc2bXyQ//xCbNpWZ4tg6Qwm5uLsFYfwhYi1MOHQx2rcvno6OylDMugefzx9yj3cCG7Bavbjd\nzcC3ULHddPr6JqHKsIyWc6T0yuWahs1mJTk5iWBQj1u7gD9hFvjZUZ+Nru0tqOlYz5v22bEjLtTA\nxAIsxVjGFXGFW1Bu/GsIBi3hF4j6+h8OeA5DubXF3S0I44+4sb4AQTgROjrclJdvorR0G+Xlz+By\nucPf6WL05z/PZ+7cacTHJwAaBw7o7nErcC3Tp2eRnNwN/A8qE/tS1HCM6cDtKOt3FZAMPAW4sNkO\nAkZXuxvVuSwdJewQGdox1Gd96EdX1D6HMQu8uYzLbu8M7Wd277tc007gCQqCMJ4Qy1oYM06mZejR\nSpaG2ieSYBaJ1WZm5vL66z6MFmvEoq4M/VKfU1LuZNOmbwJGV3sNKiFtPpFe3/XAdeidxPLz/8E5\n56Tw1lsPEAza6OvbT1/fYlR2trLwi4sT8PnSTOVn8CbgJjHxQ1avXoamESr5ysKY+Ka/QAiCMHER\nsRbGjOEI7lAcrWRpqH2ys4v4whfWsWNHHHAYny8Nl+t0Iv20zRYrmMurZs/+AmvXvk1T00cUFPhY\nsOB3vPZaGr293ai49TWhdeopK3s93EnM4bjB9BLicrmpqNBjxgEcjiux2azh8jOVld4K3ArY8Ps1\n1q5dD2CqzY6LqyQ/HzZtWjKsZyYIwvhFxFoYM4YjuDC4BR6dkd3W9i6NjbOprHw7FKtuort7CkYL\n9PDhvRw4kIjb/RNAjcMsLFwL5KFi1p+iOnzplu1HGC3xDz98h927fwxsob5+Cvn573DxxbBt23J0\nK1qNpnRTVXXLkPetu+n1+9LLuxyOEqqqluJyufnSl17G7Y5MBDPHrNXPL3zhbLZuveL4HrogCOMS\nEWthzBhux6zBLHCHoyTkEv48cASncwVf//rDIctT1Vfr61qtD5Ca6sfpXAG8gVHwsrKm09PTh9t9\nLSr+vBHoBeJJT7fS3R3pbOb1TkElhy0HOmlp6aalxQU4UAlk76EakMwIdwY7mls/+r5eeukOzjjj\ni8yceYR587yDdGTTpMOYIJyiiFgLY8ZwSog6OtzU1bWiMqe7gIXU1QUAyMs7G6cz4gJWiVadKAs5\nsj9kk5WVHJpdbSznctHR0YT6Z2BM9Ipj0qQP+dKXckNWs3EQhje0dgORUiy9i9mtqBj2tVRXR9z6\nQ8Xmoz0LXu9cdu9ewu7dkUYtA5+NlFwJwqmIiLUwZgynhGjlytqw21qJ4gbc7klUVNSGRk1GLE2b\n7QC9vS+i1y4b909N3R/6HEnqSk1tCVnincCDoTOqY71eDYvlEcrK9CYqicBpgHGKlTG+fQ7K6s4I\nb9Nd10PF5gc2V4mUiG3fHsfOnZcPsMyl5EoQTk2kdEuIaQa29vQBC2lqysThKKGsbD3nnfcsZWXr\n2bSpDKvVO+j+2dlFoX1fo6wswM6dV5KXdzaRUq5CoMh07JtvJlFVtZTSUg3l+p5i+F5PSoOI0Kah\nLHe1TXUecw8am+/ocOPz9ZCYeCeqocq9wFfDx+ovJIIgCCCWtRCj6K7j/ftbMDcoSQYmY7d7BrXM\ni4vfCrmgzfvPnNkzYF+zZbsA1S50cfjYI0eacbnchvi4RiQBbQEqLn4xSqi/Sn7+b9C0Plpbn0OP\no1dUbB7gAbDbPaxcWUtNzfdRVv2LZGZm0Nv7K/z+81Gu9oU0Nb02sg9VEIRxi4i1EJNEXMeq21hm\nppf09BaysuzMmrWeVasuoLx804A4sB4Hb2xM5fDhvUPuv2LFGezc+Qlxcb9H09rQtGyU5RwZien3\n9/GlL71McXE8mzYtYfHi/6Kt7V5UMtmnXHppOllZ7tCam3E4bmDZsrdobdXj6CrePm1aIYWFkU5q\nDsflLFv2FsYGLTNnPovdnkF19VVE9wQfbu25IAgTFxFrIWYwJmIpi7oTo5ht3fp/wvuqyVqROPCO\nHXdywQX/v717D66yvvM4/s4dSAI5QIBEuiGAEay2TC11YVxCsY0SwKBopXWkRZuV0sEx7Qw3124t\n3VBTrbZDhyJip1AqWNYkUAhVA4RWKcvWTTEqZYg0CLmS5DQJhlzI2T8eTs41yUlyDufJyef1jyR5\n8jy/x4if/G7f379QVTWelBQb+/bdicVyj5frjbra+/f/DZttKvZtXUbFskSMFd23AGeBWVitVyks\nXAgcYO7cmRQUrMAepnFxO/rorR/qPsMabMye7VhwVlv7IcYsVAuw8PqCMc8V7mvXHtA8tYgorMU8\nPM+Jfg3jPGnPbUru88A1NaMpKjIWf5WW2igpeZ709AleVl4bVcpsNvszXgVGYdTyjsGo8x2O8yEc\nsIeKitFERUW4PLOqarzHO2zYcAenTm2msXEyHR2tdHZ67iNft+6oS3GT5OTN5OU9isWS4LHCvbfj\nMUVk+NACMzEN9wBOSLjavXjMfZuSo0421/853uV7rdYZFBau6F6k1VNdbSOclwMP4Dhw4zxGr95+\nTSy1tR9y7twZl2c6/wJhr1V+773/Q2VlCq2t99HZGef1evf3nDDh1u6hbvf30l5qEQH1rMVE3Lcy\n/eu/dhET00RFxWjWrj3iUmTEfcjY4LywrAXn3qx9LrukpBqr1blKWTze64I7evUjRpyisvJx4G3g\nBSIj4/nqVyPIy3MMs3uOCuwBFpGQ8DyTJ6fS0HCW8vIUsrPfICmpvcfiJjq+UkS8UViLaTiOthxF\nQ8NZ3n13NE1NI4H5lJaOwbl2uMWSwNGjj/LUUwc5caKZrq6RjBr1X3z6adL178nEOQjtK8dd63I3\n0dLSRXGxZ4979OirhIe/CtTT1TWRq1dPYN9j3dlp429/20xj4z9Zu9Y+x96Ja489DhhDevpE4FPK\nyjZQWRlGWZmNhQt/1UPBEx1fKSLeKazFNOxBlZ2dT1mZY07Xfq6z+/ytxZJAdPQorNYngDCamowg\njI6OoqLimNeeqc3m8hG5uf9Gbu4ujh6toqnJ0eP+9NOP6ezcdP3j3TiOtQQIo7LyNpYuzae6ehoQ\nAdTg3LNPSDhDenqj28pv43urqpJU01tE+kVhLUHR2/GYnoVQjLlfb/O37td6C0LnZ9XWfkBl5SPA\nCUpLLZw6Vcirr36ZoqIyjCpmxtx3Z2eM030XERb2PDabYw82NFJdzfW2NQNfJyrqP/nsZ79w/ZeE\n5S7z0KrpLSKDobCWoOjteEz3cHPupdo5iqZ0Ar/AmKOezJkzZzl/fjqpqSlenwVZwHPAOowe8hKW\nLv0B7e3P4dqTHwf8DmNOu4nY2Gu0tPwEo6zoFYzKaP/h8j2xsVO89pg1Dy0ig6WwlqDo7XhMz3Bb\n7lIYpKHByoIFu64vLmsB/oF9q9XVqzbuv38zpaVrenyWUVrU8XFbW6rb12MxVoR/B8ee6k20tKzC\nqP8dS2Rkk8u2LIhlzhz7QjdXmocWkcFSWEvA+XIetfPQcF/h5r5PGTbjHLaNjZNdnlldfRqoAyYB\nTURHv097u+PZMTEfc/Wq4+Pw8L8QE3Mzra2OeyYm3sq8eYc5e3YkKSlW2tvDXY6wTE4u46WXHvXr\nvyNVLhMRO4W1BFxP51E79557Kh/qjWtP+Z/ANYzDMIxqYBbLRS9D369h1P22MW9eM7Gxjmd/97uZ\nfOtbRiETi+Ui+fnfIDfXtcb41KmfsnfvCurqjIM6GhutREc79/4fHVS49jYtICKisJaA8zbk7d57\ndi8f2ltYTZpUh2Pl9SGc545HjPgB+fkP88QT53Ad2o4HrEAR77wziowMG3v3Oupul5be7vKMvDxj\nq1hP88z+HtrubVpAREQVzCTgfKnK5R5Wb74J2dlv0Nho7b7GXiXs3XerMHrKBzAWejm+b8aMO0hN\nTfFS4awZo/DJclpbV7hUN3O/f0ZG8fUiLF9mz547AFi27CSf+cxmFizY79Euf1DlMhHpjXrWEnB9\nrYb2drBFa2sUhYXLgV0899yXWbfuKCUlnVitMcDNGNXGwFix7Riurq39kIwMSEq6wsKFO6iqGk9S\n0mWgg2PHYl3mod17r84nfZWWHqKk5C1Gjap2mR+/eHEPZWUr8PcwtVaMi0hvFNYyIN4WRCUmxvf6\n9Z7mdD0XjD0HrMIeqJ6lPH+CUdP7MAAjRjzDzTfPor7+LJWV36Gy0kJpqY2srF28+ebd3W2Jiemk\ntXU39pO2ej4cxCg9arWGYbXux3PPt/+HqbViXER6o7CWAfG2IMo4PrLnr/cURp5bq27FOBrTGA72\n/PoM4JcYx1oa27UmT95BRMStVFZauq9zPuXKOewTEp4nPX2i18NBjLY6lx5twbPmuIapReTGUljL\ngHhbEFVfbyU7e7+X86h774m6b+OaNOk0V69eBuppb48lKanNrUjKOVpaEl32OZ84EU56uvftYO5t\nnTLlZrZv77l4iethHwtJTt7MuHFpNDaeIyHhM0yb5jgFTFuuRORGUFjLgHjbJ716dZHP51E7y8tb\nQFvbDv7yl3CgHputDat1JRBGUZG3gy+Wc+edr2G1Ovd468nLM+a43ed9fS332dNhH/ZtWYmJD3Zv\n3bLTlisRuREU1jIg3vZJL1z4v7ifRz1lSkGfC6YslgRiYqKxWpdgzEN3YAR9JpDgtd73nDlxFBW9\nhrElq5k5c+KwWBJYv/4LLFu2n7//PYk//nEbqak3M2WKY7GZL4u3+jN/rC1XInIjKKxlQLztk25s\njMJ5fjc9PdLrcLOd8xCyMWy+H1iBo7e8B1jutSf80ktLiI4+SkXFNVJSOsnLWwzAsmX7XRarffTR\nHj76aEX3YrPBcB7m96USm4iIvyisxS+MHuV8jIAdQVTU/1FefgvZ2W/0OI9rDCHbe9MzgCqce6kj\nR3aQkbGrx+pm3nq/jY2TXe7hz9XbzsP8PVVi05YrEQkEhbX4hdHDHIOx//l3dHQ8S1lZGGVlrvO4\nnr3p/wYex3FutKOXmpFB9/nWvs4LWyyf0NoamNXb5887rxL3XolNRCQQFNbiF3l5CwgL28mxY9do\nauqgq8v7PK7nnukXcD43Oioql8jIz2CxXGTjxvsAKC8fhXNIfvzxKI/n238JGDNmOg0Nz2CzJREW\nVk1q6nTS0nb5pcebmtrMqVMa8haRG09hLT5raLDy1FN/vL5q+zJz5sTx0ktLsFgSsFgSiI6Oxmpd\njrE4zHuouS/IioyMp7PTfu0YOjpS6ej4Bq2tNnJzd7F9ewoNDX93uV99/VngHpe2uf4S8DWysnax\nffsK/Gnr1kza2jTkLSI3nsJafLZu3VEOH7YPWdsoKnqN6Oij3cPAjmHiTGDP9TlnXELNfUHWqFEN\nxMUZ+5g/+eQ8Vms29gM39u/v5NSpXxAbm4QxFx4HtDB2bIpH227EquyxYzXkLSLBobAWn3lWEoun\nouJa99cdw8QJwHIyMjznlh2FRzqxWkfQ1PQdmprGMHv2LqZOnUBh4Rjsq8BttjAqK22MGPEMsAl7\nwE+btsujbVqVLSKhTGEtPjMC0V6TOxb4gKQkxypvX4aJ7QuyMjKKKS1d2v35iorR7N17B7CLwsJm\nHD3pZq5ds7gVRfG8r1Zli0goU1iLz/LyFnDy5C+prjZqcsMSYEf31/szTOzeE66t/ZCHH4aUFBsx\nMVW0ta3u/lpExA/Yvv3fe72fVmWLSChTWIvPLJYEJk26jepqx1B4VdX4Ad3LuSdcW/uhy2lZ8fHb\naWtzPCM19TZ/NF9EZMhSWIvPvJ07PdC5YeeecEYGLqdlRURYcV79nZbWNui2i4gMZQpr8Zn7udPJ\nyZvJy3t00Pd1HxKfMyee6GjNP4uI2CmsxWfuq8EnTLgViyWhuyBJZaWF5OSGfh8T6bk4bLGOmRQR\ncaKwHqYGcg5zT9ujPKuS9e+YyIEsDtM50iIynCish6mBnMPc0/aoG3VMpHNA19Z+QGXlasCic6RF\nJOQprIcJ957oxx/H0t+A7akH3FOP29+9X9cefBbGXuyv+9x+EZGhSmE9TLj3pJOTc+mpfndf3EN4\n40ajmIkxZ93Y3eMeSO+9N54V1GKv/1kVy0QktAUsrLds2cLrr7/OuHHjAMjJyWHevHmBepz0wT3o\nxo6dwuzZA1tx3VMIJybGU1fX3OMzB9v7de/BJyeXMWFCl1aMi0jIC2jPeuXKlaxcuTKQjxAfuQfd\ntGnXBtzL9TWE+1uvu69hc88580e1qExEhoWAhrXNZgvk7aUf/Fk729cQ7u8z+xo2V0lRERmuAhrW\nu3fvprCwkNtuu43169cTHx8fyMdJL/wZdL6GcH+feaNWlYuIDDVhtkF0f1euXMnly5c9Pp+Tk8Os\nWbOwWCyEhYXx4osvUldXR25u7qAaKwNXX29l9eoizp+PIzW1ma1bMxk71lxDyA8//Dtef91Y3Q02\nvva1Pezd+/VgN0tEJOgGFda+unTpEqtWreLAgQN9Xuu8QCnUuC/AupGys/NdCpdkZfl/X/Jg36+x\n0cratUddeuxmmpMO5s8v0EL53UDvN9QNh/frS8CGwevq6khMTATgrbfeIi0tLVCPEh84hpitQBFv\nvgnZ2W+YqvKX5qRFRLwLWFj/9Kc/5aOPPiI8PJybbrqJH/3oR4F6lPjAsSisCFhOa2sYhYUD3/vs\nbeW2L78diohI/wUsrPPy8gJ1axmADRvu4NSpzVRVTcJmG/wiLm8rtwsKVviruSIi4kQVzIaJzZvf\nu3685Wv0VrnM3mMuL4+goaGCcePSmDr1isdwuVZui4jcOArrYcIRrpnAHkaO7CAjA49tV44e8x5g\nA5WVYbz/vudwube91vX1VrKz9+skLBERP1NYDxOOcE0AlpOR4Qhf5/nnf/yjEyOA43DuOZeXjyI7\nO9+jHrh95faGDV9g1qxfcfHiOvpbC1zHXYqI9E5hPUz0VsjE9TSr3RjD5M04D5dfvnyGsrKnsQdx\ne/sOfvObh7vvkZ2dz8WLtzKQoXF/H/ghIhJqFNbDRG/bolznnxeRkPA8kycn09Cw+fqc9accPZqA\ncxCfOBHu5R4tDOQkL81/i4j0TmEtbvPPY0hPn8j27fe5XJOWthXnIIZ6L/e4D2OuO5bk5DLy8h4d\nwPN13KWIiDuFtfhU63vOnDiKil4D4oFm5syJ87hHTMxhzp4dSUqKtV8nYvnzkBERkVB0Q8qN9keo\nl5Qbqu/nSynQofx+vgjl9wvldwO931A3HN6vL+pZB0ioVfhSKVARkeBRWAeIKnyJiIi/hPd9iQyE\nVjiLiIi/KKwDJCXlnxirpkErnEVEZDA0DB4gWuGsymQiIv6isA6Q/i7ICsVgU2UyERH/UFibRCgG\nm+btRUT8Q3PWJhGKwaZ5exER/1DP2iRCseSm5u1FRPxDYW0SoRhsKqQiIuIfCmuTULCJiEhPNGct\nIiJicgprERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgpr\nERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NY\ni4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NYi4iImJzC\nWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicoMK68OHD7N48WJmzpzJBx984PK1\nbdu2kZGRwcKFC/nzn/88qEaKiIgMZ4MK67S0NLZs2cLs2bNdPl9eXk5RURGHDh1i+/btPPvss9hs\ntkE1VEREZLgaVFhPnTqVKVOmeARxcXExmZmZREZGMnnyZFJSUjh9+vSgGioiIjJcBWTOuqamhqSk\npO6PJ06cSE1NTSAeJSIiEvIi+7pg5cqVXL582ePzOTk5LFiwwOv3eBvyDgsLG0DzREREpM+w/vWv\nf93vm06aNImqqqruj6urq5kwYYJP35uYGN/v5w0ler+hLZTfL5TfDfR+Q12ov19f/DYM7tybXrBg\nAYcOHaK9vZ1PPvmECxcu8LnPfc5fjxIRERlWwmyDWKb99ttvs2nTJhobGxk9ejQzZszglVdeAYyt\nW/v27SMyMpKnn36au+66y2+NFhERGU4GFdYiIiISeKpgJiIiYnIKaxEREZNTWIuIiJicacN6x44d\nzJgxA6vVGuym+NXPf/5z7rvvPpYuXcrjjz9OXV1dsJvkV3l5eSxcuJCsrCzWrFlDS0tLsJvkN73V\nwh/Kjh8/zr333ss999zDyy+/HOzm+NXGjRuZO3cuS5YsCXZTAqK6upoVK1aQmZnJkiVL2LlzZ7Cb\n5Dft7e089NBDLF26lCVLlrBly5ZgNykgurq6uP/++1m1alWv15kyrKurq3n33XdJTk4OdlP87tvf\n/jb79++noKCA+fPnh9x/gHfddRcHDx6ksLCQlJQUtm3bFuwm+U1PtfCHsq6uLjZt2sSOHTv4wx/+\nwMGDBykvLw92s/zmgQceYMeOHcFuRsBERESwYcMGDh06xJ49e9i9e3fI/Pyio6PZuXMnBQUFFBQU\ncPz48ZAsW71z506mTZvW53WmDOvc3FzWrl0b7GYERGxsbPefW1tbCQ835Y9gwObOndv9TrNmzaK6\nujrILfKfnmrhD2WnT58mJSWFm266iaioKBYtWkRxcXGwm+U3X/ziFxk9enSwmxEwiYmJzJw5EzD+\n3zJt2jRqa2uD3Cr/GTlyJGD0sjs7O4PcGv+rrq6mpKSEhx56qM9r+6xgdqMdOXKEpKQkbrnllmA3\nJWBefPFFCgsLiY+PD6lhK3f79u1j0aJFwW6G9MJbHf/3338/iC2Sgbp48SJnzpwJqQJUXV1dPPDA\nA1y4cIFHHnkkpN4NHB3T5ubmPq8NSlj3VG/8qaeeYtu2bbz66qvdnxuKvZi+6qnn5OSQk5PDyy+/\nzG9/+1vWrFkThFYOnC/14rdu3UpUVNSQmyscSC38oWwo/v0ST1euXOHJJ59k48aNLqN3Q114eDgF\nBQW0tLSwevVqzp07x/Tp04PdLL84duwY48ePZ+bMmZw8ebLP64MS1j3VGz979iyXLl0iKysLm81G\nTU0Ny5Yt4/e//z3jxo27wa0cOF/rqS9evJgnnnhiyIV1X++Xn59PSUnJkBw1GEgt/KFs0qRJVFZW\ndn9cU1Pjcx1/MYfOzk6efPJJsrKy+MpXvhLs5gREXFwcX/rSl/jTn/4UMmH93nvvceTIEUpKSmhr\na+PKlSusXbuWvLw8r9ebasI0LS2Nd955h+LiYo4cOcLEiRPJz88fUkHdl4qKiu4/FxcXM3Xq1CC2\nxv+OHz/OK6+8wtatW4mOjg52cwImVHqkt99+OxcuXODSpUu0t7dz8OBB7r777mA3y69C5WfVk40b\nNzJ9+nS++c1vBrspftXQ0NA9PHz16lVOnDgRUv+//N73vsexY8coLi7mZz/7GXfeeWePQQ0mnLN2\nFhYWFnJ/0V544QXOnz9PeHg4ycnJPPvss8Fukl/9+Mc/pqOjg8ceewyAz3/+8/zwhz8MbqP8xLkW\n/qpVq1xq4Q9VERERPPPMMzz22GPYbDYefPBBn1amDhXf//73OXnyJFarlfnz57NmzRqWLVsW7Gb5\nzV//+lcOHDhAWloaS5cuJSwsjJycHObNmxfspg1aXV0d69evp6uri66uLjIzM0lPTw92s4JGtcFF\nRERMzlTD4CIiIuJJYS0iImJyCmsRERGTU1iLiIiYnMJaRETE5BTWIiIiJqewFhERMTmFtYiIiMn9\nPyQ+uNKCpR6MAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0xa813090\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "# Plot the Data (Optional)\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "plt.scatter(inputs.numpy(), labels.numpy())\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JaFHyAG9nDET" + }, + "source": [ + "## Step 2: Define our TensorFlow variables\n", + "\n", + "We'll use Keras's object-oriented [`Dense`](https://www.tensorflow.org/api_docs/python/tf/contrib/keras/layers/Dense) layer to create our variables. In this case, we'll create a `Dense` layer with a single weight and bias.\n", + "\n", + "(**Note**: We're using the implementation of `Dense` found in `tf.layers.Dense` though the documentation link is for `tf.contrib.keras.layers.Dense`. When TensorFlow 1.4 is released, the documentation will also be in `tf.layers.Dense`) " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 34, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1505502830753, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "z9r-ZeyrXu3A", + "outputId": "6230a7a3-29fe-4d08-f101-da80425bad82" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 4, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Create TensorFlow Variables using Keras's Dense layer.\n", + "\n", + "wb = tf.layers.Dense(units=1, use_bias=True)\n", + "\n", + "# We can access the underlying TensorFlow variables using wb.variables.\n", + "# However, the variables won't exist until the dimensions of the input\n", + "# tensors are known. Once the dimensions of the input tensors are known,\n", + "# Keras can create and initialize the variables. Until then, Keras will\n", + "# report the variables as an empty list: [].\n", + "\n", + "wb.variables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "docKLUaonYG_" + }, + "source": [ + "## Step 3: Define our loss function\n", + "\n", + "Our loss function is the standard L2 loss (where we reduce the loss to its mean across its inputs)." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "0_w8ZJSCtuY7" + }, + "outputs": [], + "source": [ + "def loss_fn(inputs, labels, wb):\n", + " \"\"\"Calculates the mean L2 loss for our linear model.\"\"\"\n", + " predictions = wb(inputs)\n", + " return tf.reduce_mean(tf.square(predictions - labels))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 34, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 24, + "status": "ok", + "timestamp": 1505502830875, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "RkNbXoXkpjVH", + "outputId": "c36fc98d-3a57-4074-901d-c10ae017ae3f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u003ctf.Tensor: id=40, shape=(), dtype=float32, numpy=7.3549819\u003e" + ] + }, + "execution_count": 6, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Test loss function (optional).\n", + "\n", + "loss_fn(inputs, labels, wb)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 51, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 57, + "status": "ok", + "timestamp": 1505502830981, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "K_7beXoHOU7t", + "outputId": "1ad0856a-02ec-4117-a6c0-b41030981d87" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w: tf.Tensor([[ 1.56891453]], shape=(1, 1), dtype=float32)\n", + "b: tf.Tensor([ 0.], shape=(1,), dtype=float32)\n" + ] + } + ], + "source": [ + "# At this point, the variables exist, and can now be queried:\n", + "\n", + "w, b = wb.variables\n", + "print(\"w: \" + str(w.read_value()))\n", + "print(\"b: \" + str(b.read_value()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YIlebeb_qYtC" + }, + "source": [ + "## Step 4: Create our gradients function using `implicit_value_and_gradients()`\n", + "\n", + "With a loss function defined, we can calculate gradients and apply them to our variables to update them.\n", + "\n", + "To calculate the gradients, we wrap our loss function using the `implicit_value_and_gradients()` function.\n", + "\n", + "`implicit_value_and_gradients()` returns a function that accepts the same inputs as the function passed in, and returns a tuple consisting of:\n", + "\n", + "1. the value returned by the function passed in (in this case, the loss calculated by `calculate_linear_model_loss()`), and\n", + "1. a list of tuples consisting of:\n", + " 1. The value of the gradient (a `tf.Tensor`) with respect to a given variable\n", + " 1. The corresponding variable (`tf.Variable`)\n", + "\n", + "Test it out below to get a feel for what it does. Notice how the first value of the returned tuple (the loss) is the same as the value returned in the cell above that tests our loss function." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "v1spZQ4NwW1U" + }, + "outputs": [], + "source": [ + "# Produce our gradients function. See description above for details about\n", + "# the returned function's signature.\n", + "\n", + "value_and_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 153, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 46, + "status": "ok", + "timestamp": 1505502831114, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "21WMcpsmFFLd", + "outputId": "f51b3171-33f5-4f87-8bf7-0be2dc8edc8a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Outputs of value_and_gradients_fn:\n", + "Loss: tf.Tensor(7.35498, shape=(), dtype=float32)\n", + "\n", + "Gradient: tf.Tensor([[-3.00773573]], shape=(1, 1), dtype=float32)\n", + "Variable: \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e\n", + "\n", + "Gradient: tf.Tensor([-4.06519032], shape=(1,), dtype=float32)\n", + "Variable: \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e\n" + ] + } + ], + "source": [ + "# Show outputs of value_and_gradients_fn.\n", + "\n", + "print(\"Outputs of value_and_gradients_fn:\")\n", + "\n", + "value, grads_and_vars = value_and_gradients_fn(inputs, labels, wb)\n", + "\n", + "print('Loss: {}'.format(value))\n", + "for (grad, var) in grads_and_vars:\n", + " print(\"\")\n", + " print('Gradient: {}\\nVariable: {}'.format(grad, var))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JVDWpL9VYWdP" + }, + "source": [ + "## Step 5: Create an optimizer\n", + "\n", + "We'll use a `GradientDescentOptimizer` to fit our model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "DudNEebMKDWN" + }, + "outputs": [], + "source": [ + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YBeJYxY8YaiO" + }, + "source": [ + "### Step 5a: Test Our Optimizer\n", + "\n", + "Now we have everything needed to start fitting our variables to the data!\n", + "\n", + "In the next cell, we'll demo these capabilities. We'll:\n", + "\n", + "1. Print the current values of `w` and `b`\n", + "1. Calculate the loss and gradients\n", + "1. Apply the gradients\n", + "1. Print out the new values of `w` and `b`\n", + "\n", + "You can run the cell multiple times. Each time, you should see the values of `w` and `b` get closer to their true values of 3 and 2." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 102, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 103, + "status": "ok", + "timestamp": 1505502831285, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "diDZfrMJM3OC", + "outputId": "d585fff0-ecb3-4e98-9b33-bbae07a95d8c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Values of w, b, BEFORE applying gradients:\n", + "(array([[ 1.56891453]], dtype=float32), array([ 0.], dtype=float32))\n", + "()\n", + "Values of w, b, AFTER applying gradients:\n", + "(array([[ 1.86968815]], dtype=float32), array([ 0.40651903], dtype=float32))\n" + ] + } + ], + "source": [ + "# Test the optimizer.\n", + "\n", + "print(\"Values of w, b, BEFORE applying gradients:\")\n", + "w, b = wb.variables\n", + "print(w.read_value().numpy(), b.read_value().numpy())\n", + "print()\n", + "\n", + "# Calculate the gradients:\n", + "empirical_loss, gradients_and_variables = value_and_gradients_fn(\n", + " inputs, labels, wb)\n", + "optimizer.apply_gradients(gradients_and_variables)\n", + "\n", + "print(\"Values of w, b, AFTER applying gradients:\")\n", + "print(w.read_value().numpy(), b.read_value().numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "61TgeLVlKEQp" + }, + "source": [ + "## Step 6: Create a training loop\n", + "\n", + "Of course, now we can simply turn all of this code into a self-standing training loop. We'll also capture our loss and approximations of `w` and `b` and plot them over time." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 397, + "output_extras": [ + { + "item_id": 1 + }, + { + "item_id": 2 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 225, + "status": "ok", + "timestamp": 1505502831550, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "VukGe-huNaJ4", + "outputId": "f0a8d665-1910-477c-d8ab-c94ccdc4afcd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2.111051321029663, 2.3047544956207275, 2.4602210521698, 2.5850086212158203, 2.6851789951324463, 2.7655951976776123, 2.830157995223999, 2.8819968700408936, 2.9236228466033936, 2.9570505619049072]\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAFXCAYAAADnFpTQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFUbBfAzu+m9koSShBQCSC+igIAgRRGkChJEiggo\nHURAEBQBQeADRcWCha50ULFLk6IivYRQQwskhPS6O/P9sckmm4Rkk2x2difn9zz7bLuZvC8JHO7M\n7FxBkiQJREREVOlUchdARERUVTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMjArdlJQU\njB8/Hk8//TS6d++OkydPVnZdREREiiMY8znd6dOno2XLlujbty80Gg0yMzPh4uJijvqIiIgUo9TQ\nTU1NRa9evfDbb7+ZqyYiIiJFKnX38s2bN+Hp6YkZM2agd+/emD17NjIzM81RGxERkaKUGroajQbn\nzp3DoEGDsH37djg4OOCzzz4zR21ERESKUmro+vv7w9/fHw0bNgQAdO3aFefOnSvxa3g5ZyIioqJs\nShvg4+ODgIAAXL16FbVr18aRI0cQGhpa4tcIgoC4uBSTFSkHX19Xq+8BUEYfSugBYB+WRAk9AMro\nQwk9ALo+jFFq6ALArFmzMHXqVGg0GtSqVQsLFy6sUHFERERVkVGhW7duXWzdurWyayEiIlI0XpGK\niIjITBi6REREZsLQJSIiMhOGLhERkZkwdImIiMyEoUtERCbRuXM7uUuweAxdIiIyCUEQ5C7B4hn1\nOV0iIqKy+OijFTh69BAEQYUhQ4ajU6fOuH8/HnPmzER6ehq0Wi2mTJmOJ59sgwUL3kZU1HkAArp3\n74nnn39B7vIrDUOXiEhh5s6dhd27d5h0mz169MLcue8aNXbv3t9x+XI01qz5Fg8eJODll4egadNm\n+PXXn9Cq1eN48cVhkCQJmZmZOH/+POLi7uGbbzYBANLSUk1at6Xh7mUiIjKp06dP4qmnugIAPD29\n0LRpc5w/fw716j2CH37Yha+++hyXLkXD0dERtWrVwp07t7F8+RIcPXoYTk7OMldfuTjTJSJSmLlz\n3zV6VloZCq80l/e8ceOm+Oijz3H48EEsWDAXAwcOxuDBA/D11xtx9Ohh7Ny5DX/88StmzHhLjrLN\ngjNdIiIyifxwbYbff/8VoijiwYMHOHXqBOrXfwSxsbHw8PDEs8/2wrPP9sLFixeQmJgIUdSiffsn\n8fLLoxEdHSVzF5WLM10iIjKJvLOX27d/EmfPnsbQoS9AEFR49dXx8PT0wp4932PjxrWwsbGBk5Mz\nZs16G7GxsXj99TcgSSIEQcDo0eNk7qJyCVIlrThv7esjKmmNR2vvQwk9AOzDkiihB0AZfSihB8D4\n9XS5e5mIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIismjHjx/DmTOn9M93\n7NiKn3/+0STbXrv2K5Nsx1gMXSIismjHjx/D6dP5odurV1907fqMSba9Zo15Q5dXpCIiogrbsGEN\n7O3t0bfvAHzwwVJcvnwJK1Z8gmPH/sGPP+7C7NnzDMZHRV3Ahx8ug0aTDWdnN7z55hx4eXlj8+ZN\n2LlzG2xsbBAcXBujR4/Fzp1boVbb4Ndf92DixNfx779/w8nJCQMHDsa4caNQp04ETp48gczMTMya\nNRdr136FK1cuo2PHzhg5cgwAYMaMqYiLu4fs7Cz07/8CevTohVWrViI7OwvDh0eidu0QzJ49D7/8\nsgebN2+CVqtB/foNMGXKdJOuE8zQJSJSGOe5s2Bv4qX9snr0QloJiyg0btwM3367Hn37DkBU1AXk\n5ORAq9Xi1KkTaNy4mcFYjUaD5csX4733liEsrBY2bdqGTz/9CDNmvIX167/Bli27YWNjg7S0VDg7\nu+C55/rqQxYA/v33b4Pt2dra4Ysv1mDz5k2YPn0KvvpqPVxcXDFgQC8MGBAJNzc3zJw5B66ursjK\nysLIkUPQvn1HjB49Ftu2bcaXX64HAFy/fg2///4LVq36Emq1GkuXLsIvv+wx2awaYOgSEZEJRETU\nRVTUeaSnp8PW1hYREXVx/vw5nDx5HJMmTTMYGxNzHVeuXMakSa9BrVYhO1sDHx9fAEBYWDjmzn0T\n7dp1wBNPdDDqe7dt2w4AEBoahpCQUHh6egEAqlevgXv37sLNzQ3ffbcBBw7sAwDcu3cPN2/GoH79\nBgYrIv3779+4eDEKI0cOgSRJyM7OhpeXV0X/aAwwdImIFCZt7rslzkorg42NDfz9A/Djj7vQsGFj\nhIWF4/jxf3H79i0EBQUXGi0hJCQUn3zyZZFrL7///gqcOPEfDh7cjzVrvsSaNd+W+r1tbe0A6BZc\nsLW11b8uCAK0Wi2OHz+G//77F5999jXs7OwwbtwoZGdnF7MlCd26dceoUa+V40/AODyRioiITKJx\n46bYuHEdmjRphkaNmmDHjq0ID69TZFxgYDAePEjEmTOnAeh2N1+9egUAcPduLJo2bY4xY8YhLS0N\nGRnpcHJyQlpaWrnrSktLhaurK+zs7HD9+jWcPXtG/56trS20Wi0AoHnzR7F37+948OABACA5ORmx\nsbHl/r7F4UyXiIhMonHjpli79is0aNAQ9vYOsLe3L3I8F9DNit99dxGWL38fy5cvQnZ2Dp5//gXU\nqhWId96ZnRuwEvr3HwhnZxe0adMOs2a9gb/+2o+JE183OLGppJOc8t5r1ao1duzYisGDn0dgYBAa\nNGioH9OzZ2+89NJARETUxezZ8/Dyy2MwefJrEEUJtra2mDx5Gvz9/U32Z8Sl/R5CSctNWXsfSugB\nYB+WRAk9AMroQwk9AFzaj4iIyOIwdImIiMyEoUtERGQmDF0iIiIzYegSERGZCUOXiIjITBi6RERk\ndt99txFZWVlyl2F2DF0iIjK7zZs3Iisrs9j3RFE0czXmw9AlIqIK27BhDbZu1V0n+YMPlmLCBN2S\neseO/YN582YbjN2yZRPi4+MwbtxovPTSSwCAzp3bYeXK5Rg2bBDOnDmF/v17Ijk5CQBw4cJ5jBs3\nCgCQmZmJhQvfwciRL2H48ME4eHC/uVo0CV4GkohIgbyaNyj29YRjZ4p9vazjCyvL0n79+g3Et99u\nxIcfforQ0BqIi0tBZmYGGjRoiLFjJ+aOMry8Y94lHb/5ZjWaN38UM2a8hdTUVIwcOQQtWz4Ke3sH\no+qUG0OXiIgqrCxL++lIuTcdtVqN9u07Fnq/qH/+OYpDhw5g48Y1AHSLJdy9G4vAwGCT9VKZGLpE\nRApk7Ay1vOMLK9vSfkXZ2dkbLF6gVqshirrgzc7OP+FKkiS8++5i1KoVWKF65cJjukREZBLGLu0H\nAE5OzgbL9RVeeycgoDqios4DAPbt+0P/+qOPPoYtWzbpn0dHR5myhUpn1Ey3Y8eOcHFxgUqlgo2N\nDbZs2VLZdRERkZUxdmk/AOjZsxemTh2PgAB/LFmyssgSfUOHjsR7770DFxcXNG3avMDrL+ODD5bi\npZcGAgD8/QOwaNH/Kq8pEzNqab9OnTph27ZtcHd3N2qjFy9ehKdnQIWLk5OSlpuy9j6U0APAPiyJ\nEnoAlNGHEnoATLy0nyRJZfrc1IABA5CTk2P0eCIioqrAqNAVBAEjRoxA37598d1335U6/sSJE/jw\nQ+uZ7hMREZmDUcd0N23aBF9fXyQkJGDYsGEICQlBixYtHjq+Ro0aWLp0Ebp164769R8xWbFERETW\nzKhjugWtXLkSzs7OGDZs2EPH/PDDD3j22WfRvHlzHDlyBDY2/GQSERFRqWmYkZEBURTh7OyM9PR0\nHDx4EGPHji3xa7p3747nn38B3323EXPnvosJE6aYrGBzUdLBfWvvQwk9AOzDkiihB0AZfSihB8D4\nE6lKDd34+HiMHTsWgiBAq9WiR48eaNu2bakbfvfd97Bv3594//2F6NatOyIi6hpVEBERkVKVeiJV\nrVq1sHPnTuzYsQO7d+/GK6+8YtSGPTw88f77y5GdnY0JE8ZAo9FUuFgiIrJMsbF3MGTIAJNuMzr6\nIg4f/kv//ODB/Vi//huTbFuupQUr9YpU3bo9g759n8d//x3DqlUfVea3IiIimRW+wEVFXbp0EUeO\n5Idu27btEBn5kkm2XdLSgpWp0s9wmj9/Efbv34tFi95F165PP/SSYEREZN00Gg3eeWc2Ll68gNq1\nQzFr1tuwt7c3GHPr1k0sW7YYSUmJcHBwwHvvLYCLiw/++OM3fP3151Cr1XB2dsHy5R/jiy9WITs7\nG6dPn8TgwcOQlZWJCxfOYdKkaViw4G3Y2dkjOjoKiYkPMGPGW9iz53ucPXsa9es3wMyZcwAAS5a8\nh6ioc8jKykKHDp0wfPgrBksLenh4YMWKT/D330fw5ZefIScnBzVq1MTMmXPg4GD6lYsqPXS9vLyx\nePH/MGxYJCZMeBW7d/8MtVpd2d+WiKjKmjvXHrt3m/af9x49NJg7t+TdsTEx1zFjxhw0aNAQCxe+\ng+3bN2PgwMEGYxYvXoBp02aiRo2aOHfuDObOnYslS1bim2++wLJlH8HHxwdpaamwsbHByy+PRlTU\neUyc+DoAYM+e7w1m06mpKfj0069w8OA+vPHGJKxa9RVq1w7BiBEv4tKlaISFhWPUqNfg6uoKURQx\nYcIYXLlyyWBpQTc3NyQlJWLNmi+xYsXHsLd3wPr132DTpnUYOvRlk/4ZAmZaZah79x7o1asPduzY\nhs8//wSjR5d89jMREVkfPz9/NGjQEADQtesz2LLlW4PQzcjIwJkzJzF79hsFFjjQ3Tds2Bjz589B\nx46d0b79k0Z9vzZtngAAhISEwcvLG7VrhwAAatcOQWzsbYSFheP333/Grl07oNVqkZBwH1evXkVI\nSBgKLi149uwZXLt2BWPGjIAkSdBoNGjQoFHF/0CKYbYP0C5YsAQHD+7HggXvoEuXbrlNExGRqc2d\nm1XqrLQyFD6mW/gQrySJcHV1w5dfrte/lveRoalTZ+D8+bM4dOggRox4EatXryv1+9nZ2QEAVCqV\n/nHec61Wizt3bmPTpvVYvXotnJ1dsGDB2wbLBObXJaFly8cwZ867ZWm3XMy2tJ+Pjw/ee28pMjMz\nMWHCa2W6ljMREVm+2Ng7OHtWty7vr7/+jEaNmhi87+TkjICA6vjzz9/0r124cAGA7lhvvXqPYMSI\nUfDw8MS9e3fh5ORksPxfSYq7zlNaWhocHR3h5OSMhIT7OHLkkEEtedt+5JGGOH36JG7dugkAyMrK\nxI0bMWXo3HhmvVRUz5690aPHduzevQOrV3+KkSPHmPPbExFRJQoKCsa2bd9h4cK3ERwcgl69+hUZ\nM2fOu3j//YX45psvodVq0LNnD/Tv/yI+/ngFbt68AQBo3rwlwsLCUa2aH9at+xrDh0di8OCHXwUR\nKP7M6bCwcISHRyAysh+qVfNDo0aN9e/lLS3o4+OLFSs+wcyZczB37kxkZ+dAEASMHDkGtWoFVvBP\npJg6y3oZSGM97AojcXFxeOKJlsjMzMSffx7S74O3NEq6Soq196GEHgD2YUmU0AOgjD6U0ANg4qX9\nTMnX1xcLFy5Beno6Jk0ay93MRERUZZg9dAGgV6++ePrpZ3Ho0EF8/fVqOUogIiIyO1lCVxAELF78\nP3h4eOCdd97C9evX5CiDiIjIrGQJXQDw8/PD/PmLkZ6ehsmTxxV75hkREZGSyBa6ANCv3wB06dIN\nBw7sw5o1X8lZChERUaWTNXQFQcCSJSvg7u6BuXNnVdrnooiIiCyBrKELAP7+AZg3byHS0lK5m5mI\nyEoZu7Tfnj3f4/79eDNUZJlkD10AGDBgEDp16ox9+/7Ehg1r5S6HiIjKwZil/X78cTfi4uKKfa8q\nfITUIkJXEAQsXfoBXF3d8NZbM3H79i25SyIiojLKW9pv8OD+mD17epFF4vfu/R0XLpzHvHmzMXx4\nJLKystCxY0d88smHGDHiRfz5528YN24UoqJ0l4ZMSkpE//49AegC+eOPV2DkyJcwdOgg7Nq13ez9\nmYJFhC4AVK9eA++8swApKcmYMmU8dzMTEVVA8+bOxd5MNb44MTHX0afP81i3bjOcnJywfftmg/c7\ndOiEevXqY86cd/Hll+v1a+26u3tg9eq16NSpSzFb1c2ev/9+J1xcXPH559/g88+/wa5d2xEbe6dM\n9VkCiwldABg06EV06NARv//+K779doPc5RARURkUXtrv1KmTRcZIkoTCc6pOnTqXuu2//z6Cn376\nAcOGDcIrr7yE5OQkqzz51qwLHpRGEAQsW/Yh2rV7DLNnz0CHDh3h7x8gd1lERFbn2DHjVucp7/ji\nlLa038M4OjrqH6vVakiS7thudnZ2gVESJk16HS1bPlbRMmVlUTNdAKhZsxbmzJmHpKRETJ06gbuZ\niYisRGlL+wGAs7Mz0tJSH7qNgIAauHDhHAAYLAH46KOPY9u2LdBoNACAGzdikJWVacryzcLiQhcA\nhgwZhieeaI9ffvkJW7Z8K3c5RERkhLyl/QYP7o+UlORil/Z7+ulnsWTJQv2JVIVnxy+8EInt27di\n+PDBSE5O1r/eo0cvBAfXxogRgzFkyAAsWbIQWq220nsyNbMv7WesmJjraNfuMdjZ2eLAgX/g5+dn\nosqMo6Tlpqy9DyX0ALAPS6KEHgBl9KGEHgALXtrPWIGBQZg9+20kJiZi2rRJ3M1MRERWz2JDFwCG\nDXsZrVu3xZ4932PHjq1yl0NERFQhFh26KpUK//vfSjg5OWHGjKm4d++e3CURERGVm0WHLgDUrh2C\nN9+cg4SEBMyYMVXucoiIiMrN4kMXAEaMGIVWrR7H7t07rPbSX0RERFYRuiqVCitWfAQHBwdMnz4F\n8fFVd4UKIiKyXlYRugAQEhKGGTPeQnx8PGbO5G5mIiKyPlYTugDwyitj0KLFo9ixYxt++GG33OUQ\nERGViVWFrlqtxooVH8Pe3h7Tpk1CQsJ9uUsiIiIymlWFLgCEh9fBG2/MQlzcPbz55htyl0NERGQ0\nqwtdABgzZiyaNWuOrVu/w08//Sh3OUREREaxytDV7Wb+BHZ2dnj99YlITHwgd0lERESlssrQBYCI\niLp4/fUZuHs3FrNnz5C7HCIiolJZbegCwGuvTUDjxk3x7bcb8OuvP8ldDhERUYmsOnRtbGzwwQef\nwNbWFlOnTkRSUqLcJRERET2UVYcuANSrVx+TJ0/DnTu3MWfOm3KXQ0RE9FBWH7oAMH78ZDRo0Agb\nNqzFH3/8Jnc5RERExVJE6Nra2uKDDz6BjY0NJk8eh5SUZLlLIiIiKkIRoQsADRo0xMSJU3H79i3M\nnTtb7nKIiIiKUEzoAsDEiVNRv34DrF37Ffbt+1PucoiIiAwYHbqiKKJ3794YPXp0ZdZTIXZ2dvjg\ng4+hVqsxefI4pKamyF0SERGRntGhu2bNGoSGhlZmLSbRqFETjB8/CTduxGDevDlyl0NERKRnVOjG\nxsZi37596N+/f2XXYxKTJ7+BunXr4auvvsDBg/vlLoeIiAiAkaG7YMECTJs2DYIgVHY9JmFvb48V\nKz6GSqXCxIljkZaWJndJREREsCltwN69e+Hj44N69erh6NGjRm/Y19e1QoVVVJcuHTBt2jS89957\nWLZsAT744IMyb0PuHkxFCX0ooQeAfVgSJfQAKKMPJfRgLEGSJKmkAcuWLcOuXbugVquRlZWFtLQ0\ndO7cGYsXLy5xw3Fx8p/ElJmZiaeeegIXL0Zh5849ePzxNkZ/ra+vq0X0UFFK6EMJPQDsw5IooQdA\nGX0ooQfA+P84lLp7efLkydi7dy9+//13LFu2DK1atSo1cC2Fg4MDli//CCqVChMmvIr09HS5SyIi\noipMUZ/TLU6LFo9i9OixuHbtKhYunCd3OUREVIWVKXQfffRRrFq1qrJqqTRvvPEmQkPD8NlnH+Po\n0SNyl0NERFWU4me6AODo6Ijlyz8GAEyc+CoyMjJkroiIiKqiKhG6ANCq1WN45ZUxuHz5EhYtmi93\nOUREVAVVmdAFgBkz3kJwcG2sWrUS//77t9zlEBFRFVOlQtfJyQkrVnwMURQxYcKryMzMlLskIiKq\nQqpU6ALA44+3wcsvj0J09EUsWfKe3OUQEVEVUuVCFwDefHMuAgODsXLlchw/fkzucoiIqIqokqHr\n7OyM5ctX6nczZ2VlyV0SERFVAVUydAGgbdt2GDp0BC5cOI///c86rrBFRETWrcqGLgC89dY7qFUr\nECtWLMOpUyfkLoeIiBSuSoeui4srli37EFqtFuPHv4rs7Gy5SyIiIgWr0qELAO3bP4kXXxyGc+fO\nYPnyJXKXQ0REClblQxcA5s6dhxo1amL58iU4c+a03OUQEZFCMXQBuLq6YenSD6DRaDB+/Bjk5OTI\nXRIRESkQQzdXx45PYdCgF3HmzCl8+OH/5C6HiIgUiKFbwNtvz4e/fwCWLl2E06e5m5mIiEyLoVuA\nu7sHli5dgZycHAwdOhSpqalyl0RERArC0C2kc+duiIwcgv/++w8DBvRGcnKS3CUREZFCMHSL8f77\nyzFo0CD8889R9O3bEwkJ9+UuiYiIFIChWwwbGxusWbMGgwa9iJMnj6N372cRFxcnd1lERGTlGLoP\noVarsWzZhxg+fCTOnz+LXr2exp07t+Uui4iIrBhDtwQqlQoLFy7Bq6+OR3T0RfTs2Q03bsTIXRYR\nEVkphm4pBEHAnDnzMGXKG7h+/Rp69uyGK1cuy10WERFZIYauEQRBwBtvvIlZs+bi1q2beO65pxEV\ndUHusoiIyMowdMtg/PjJmD9/Ee7ejUWvXk/j9OlTcpdERERWhKFbRiNHjsGSJSuQkJCAPn2exfHj\nx+QuiYiIrARDtxyGDBmGDz9chZSUZPTt2xNHjhyWuyQiIrICDN1yev75F/DZZ18hMzMDAwf2xoED\n++QuiYiILBxDtwJ69uyNr75aD41Gg0GD+uG3336WuyQiIrJgDN0K6tr1aaxb9x1UKhVeemkQfvhh\nt9wlERGRhWLomkCHDh2xceNW2NnZ4+WXh2Dbts1yl0RERBaIoWsirVu3xebNO+Ds7IIxY17Ghg1r\n5S6JiIgsDEPXhFq0eBTbtu2Gp6cnJk58DatXfyZ3SUREZEEYuibWqFETbN/+I3x9q2HGjKn4+OMP\n5S6JiIgsBEO3EtSrVx87d+5BQEB1zJ37JpYuXQRJkuQui4iIZMbQrSRhYeHYuXMPAgODsGjRfCxY\n8A6Dl4ioimPoVqLg4NrYuXMPQkJCsWLFUsyePZ3BS0RUhTF0K1mNGjWxc+dPqFu3Hj777BNMnToR\noijKXRYREcmAoWsGfn5+2L79RzRo0Ahr136FceNGQ6PRyF0WERGZGUPXTLy9vbFt2240b94Cmzdv\nwujRI5CTkyN3WUREZEYMXTPy8PDE5s078fjjbbBr13YMHz4YmZmZcpdFRERmwtA1MxcXV2zcuBXt\n2z+Jn3/egyFDBiI9PV3usoiIyAwYujJwcnLC2rXfokuXbti79w8MGtQPqakpcpdFRESVrNTQzc7O\nRv/+/dGrVy/06NEDK1euNEddiufg4IAvv1yHHj164dChg+jfvxeSkhLlLouIiCqRTWkD7OzssGbN\nGjg6OkKr1eKFF15Au3bt0KhRI3PUp2h2dnb49NMvYW9vjy1bvkWfPj3w3Xc74O3tLXdpRERUCYza\nvezo6AhAN+vlR11My8bGBitXfooXXxyK06dPok+f7rh7967cZRERUSUodaYLAKIook+fPoiJiUFk\nZGTps9zgYHiJRa+8lHDsTLHDvZo3KPZ1WcerhCI9VGY9XwFwGDkan3++Cr16PY2tW3ejevUaFd9+\ngT6s6s+/oNweLKaeco5HzHWLqofjOd4SxisiL4CH/v0uzKjQValU2LFjB1JTU/Hqq6/i0qVLCAsL\nK/Fr1CqhyGu+vq4P+QZFx1rC+MI9VHY9n376Mby83LFo0SL07v0M/vjjDwQHB1d4+3l9yP3nWZHx\napVgUfWUZ/xDv8ZK6i843uBrLaCe8ozXP7eQeso7vrh/a+Wsp8zjoYy8MJYglfFiwCtXroSzszOG\nDRtW4ri4OOs+G9fX11WWHiRJwtKli7B48QJUr14D27btRkhIyf/BKYlcfZiSEnoA2IclUUIPgDL6\nsPgeRBHIzISQmQEh9x4ZmRCyMiFkZgKZGRAyMuE+dJBRmyt1ppuQkABbW1u4uroiMzMThw8fxiuv\nvFLhPqh4giBg6tTpcHBwxDvvzEbPnk9jy5ZdqFu3ntylERHJy8gAzHtfN7bg89z3szLzt5M7Hpm5\n28nIHZf3fna2cbWZKnTj4uIwffp0iKIIURTxzDPPoH379sYVQeU2duwEODo6YMaM19G79zP47rsd\naNiwsdxlEREVJUlAdjaE9DQI6em5tzT9PdLTIaQ95D1JA9fEFMMAzMrSPy9XAJa1fEEAHB0hOThA\ncnCE5OICyccXkoM9JAdHIO91BwdIjo6Avb3hcwcHuBj5vUoN3YiICGzfvr2CLVF5jBgxCg4Ojpg8\neRz69OmBTZu2onnzlnKXRUTWSJKAjIwioWd4nw4UfC2taEgWHZf7ulZb7tIcCpZZXAB6+0BydCga\ngA4ORQPRwQGSfe57jo4FxjoCDvaGz/O2aWsLCGU7NluYyUKX5BUZOQQODg4YO3YU+vV7Dhs2bMbj\nj7eRuywiqkyiqAuylBQIqakQUpINH6emQJWaCkg5cI5/UCQQ9Y/T8h8jIx2CCdbzllQqSE7OkJyc\nACcniN4+kJyc9K9JTk6QnAs8dnIGDN43fM+rpi/i00VdANo7AHZ2FQ5AS8bQtQJ9+z4POzt7jB49\nHAMH9sGaNZvQvv2TcpdFRAVJku64YEpKbiimFB+aqbrHKv17KbrX8h6npEBISzU6IJ2KK8XWVh9u\nors7pIDqucH38PArGJYlhSTs7U0bir6ukCz5RCoTY+haiR49noODw3oMH/4iBg9+HqtXr0GXLk/L\nXRaR9cvJyZ095oeeKi0lPwALhmZaam5gFgzRlPyvL+fFgyQ7O0iurpBcXCEGBUN0dc197gLJxS3/\nsasrJFfc0+i4AAAgAElEQVQ3iC4ukFxc4FHTDwlZAJxzw9HRUReMtram/TMik2HoWpHOnbth/frN\nGDJkIIYOjcSnn36JHj16yV0WkbxEEUJyEoTERKiSEiEkJkJISoQqMbH415ISgdRkeCcl6YKynMtr\nSmo1JBddOIoB1SE560JRdHXLD0gXV/2Y/OB0g+iS/1hycdHNHsvD1xXaKjRLVAKGrpVp164DNm3a\nhkGD+mPkyKH48MNV6N9/oNxlEVWMkcGpSnxQJECF5KQyHauUnJwAd3eInl6QagUWmUnqQzMvLAuG\npqsrRGfdPRwdFX3skSoHQ9cKPfZYa2zZshMDBvTB2LGjkJWVhcGDX5K7LKrqSgzOB/qQzAvSigan\n6O4BsXp1iPXqQ/LwgOTuAbHQveThAdHDE5KHJ0R3D0ju7oC9PXx9XfGAM0SSAUPXSjVr1gLbtn2P\n559/DpMnj0NmZgZefnm03GWRUmRkQBUfB9X9eKjux0OIj4fq/n2o7scDmalwi40zTXB6eEKsXgNi\n/UfyQ1Iflh6FXvPUvwc7u0psnqjyMHStWMOGjbBjxx707dsDM2dOQ0ZGJsaNmyh3WWSJ0tL0AaoP\n0fgCz+/H54bsfaji43UXLShB3hFIyckZoocHg5PISAxdKxcRURe7du1B3749MW/eW8jISMfrr8+A\nwGNNyiVJhiEaHwchNyzzn+cFqm52KqSnl75Ze3uI3j7QhIVD8vaG6O2ju/n4QPLxzX3uDc/QWojX\n2up21TI4icqEoasAISFh2LlTN+NdsuQ9ZGZmYvbstxm81kKSdB9FiYszDMr4eMNdvPfv658bc8at\n5OCgC9HwiEIh6gvJx0cfonnPJWcX404MqmKfqyQyJYauQgQGBmHXrp/Qt28PrFy5HBkZ6Zg/f7Hc\nZVVdkqQ7eSg2Fqo7t6G6GwukJcL5+q1Cx0lzH2dllb5JR0eIPr7Q1K2nuwpQboDqZ6O5AZoXrnB2\n5tm1RBaGoasgAQHVsWPHHvTv/xxWr/4MWVlZ+Prr1XKXpTzp6VDF3oH6bm6g6oP1DtR37kAVeweq\nu7HFzkYLXj1IcnKG6OMDTf1HdCFaIDAfGqJEZNUYugpTrVo1bN/+PQYM6IN1677BvXt3MG/eYtSu\nHSJ3aZZPo4Hq3l1daBYIT/Wd27rHsXd0AZuU+NBNSCoVRN9qutmof4D+pg2oDrewIDywdc4PUafi\nLuBHRErG0FUgLy9vbN26C6+8Mgy//PIL9u/fjwkTpmDs2ImwL++Vb6yZJEF4kKAL0oKz0dhYqGIL\nzFTj7pX4kRfRwwNiQAA0TZvlBmkARL8AiAHVIfr76+59fAGbh/y18nWFhsdCiao0hq5Cubm5Y+PG\nrfjzzz2YMGEiFi2ajy1bvsWiRcvQrl0HucsznbQ0qO8WmJnmBqsqNm+GGgvV3TslHjOVHBwg+gcg\np9XjEIsJUq2fP0T/AN0ViIiIKoChq2CCIGDAgAFo0aINFi2aj9WrP0O/fj3Rp08/vP32Qvj5+cld\n4sPlnoikvhEDJMXB4eKV/BlqgWBVJSc9fBMqFUQ/f90xU78AXaDm7uoV/fz1wSq5e/CEIyIyC4Zu\nFeDm5o758xdjwIBBmDZtErZt24Jff/0FM2fOxtChL0OtVstTWFoa1DdioI65BlXMdaivX4c6RndT\nxVyHKiVZP9S10JeKnp4Qa9SEpnkLaP0Dip2hij6+gFy9EREVg6FbhTRq1AQ//PAb1q79GvPnv40Z\nM17Hpk0b8P77/0OTJs1M/w2zs6G6dVMfpLowvaZ7fP06VPFxxX6Z5OQEbWAQcgJbQxsYBKd6dZDs\n6gWtf26g+gcADg6mr5eIqJIxdKsYtVqNoUNH4JlneuDtt2dh8+ZN6Nr1SQwdOgIzZ74Fd3cP4zcm\nirqPzsRch+r6NYNZqjrmOlR3bkMQxSJfJtnaQluzFjSPNIA2MAjawCCIuffawGBIPj4Gu3udfF2R\nxROQiEgBGLpVVLVq1fDRR59h0KAXMW3aJHz11Rf4/vtdePvt+ejb93nd1awkCUJCAtS5s1OVfvdv\n7u7gmzcgZGcX2bYkCBADqiPn0ccKhGkQxKBg3b1/AHf7ElGVxNCt4to2boIDH32OXz/7GCd3bkPW\nqyNxcdZ0NPX0hGNsLFRpqcV+nejjkztTDS4UrEHQ1qhV/kW5iYgUjKGrdFlZUF+OLjBLzdv9mzt7\nTUgAAAzOvQEAEu4jOeE+Yn184dGmLVA7JDdYdTNVba1AwMVFro6IiKwWQ1cJJAlCfDxsoqOgvhgF\ndXQUbC5GQX0pGrh9C17FXPBBsreHtlYgNI2b5odpkC5Qf4m+iCnz38btO7cReOEC3hs6Ak891VWG\nxoiIlIWha01EEapbN2Fz8QLUFy/mh2t0FFQPHhQZrq1eA2jfHhkBNQ1OVBKDgiBW8wNUqmK/Taem\nzXHwmR5YunQRPv30Iwwa1B/du/fEu+++hxo1alZ2l0REisXQtUQ5OVBfvQL1xagCs9eLsLl0sci6\nqJJKBW1wbeS0ehza8Aho6kRAWycC2vA6kFxc4evritRynPnr4uKCOXPm4fnnX8C0aZPwww+78Oef\nv2PatJkYOXI0bG1tTdUtEVGVwdCVU1oabC5dzA/V3Fmr+uoVCBqNwVDJwQHa0HBo6tTJD9fwCGhD\nQiv1pKV69epj5849+PbbDXj77VmYO/dNfPvtBixe/D+0avVYpX1fIiIlYuiagXD/ftHjrdEXob55\no8hY0d0DmibN8kO1Th1owiMg1gqU7WM2KpUKL7wwGF27Po13352Ldeu+QY8eXRAZOQSzZ78NLy9v\nWeoiIrI2DF1TkSSobt8qsEv4ItQXL8AmOgqq+/eLDNf6+SP7iQ76UNXWiYAmPAJStWoWex1gLy9v\nLFv2IQYMiMS0aZOwfv0a7NnzPd56ax4GDoyE6iHHiImISIehW1YaDdTXrhaatUZBHR1d5DOtkkoF\nMTAIWc1bFtglXAfaOhGQ3NxlaqDiWrV6DL/9th9ffPEpFi2aj4kTX8OGDWuxePH/UL/+I3KXR0Rk\nsRi6D5OeDpvTJwuEq+5sYfWVyxBycgyGSnZ20IaGI7tAqGrCI6ANDVPsNYJtbW0xZsxYPPdcb8ya\nNR3ff78TnTq1xahRr2Hq1Olw4ed4iYiKYOgCEFKSYXPqJGxOnoDNqeOwOXkCuHIZnoU+3yq6uELT\nsBG0deoW2CVcB2JQcJW9rGH16jXw5Zdr8dtvP2P69Nfx8ccfYMeOrZg/fzGeeeZZ3eUkiYgIQBUM\nXSE5CTanT+kC9uR/uvsrlw3GiG7uQLt2yKgdVuCEpgjdNYMZIsV66qmuOHCgHVasWIIPP1yOYcMi\n0blzVyxY8D6CgoLlLo+IyCIoOnSF5KQiM9giAevugewn2kPTqAk0TZoip1ETiMG14VvNrVyfb63K\nHB0dMX36bPTtOwBvvDEZv/76Mw4e3I9Jk17Hq6+Oh52dndwlEhHJSjGhW6aAbdwUmsZN9AHL2atp\nhYfXwdatu7F163eYM+dNLFjwDjZv3oRFi5ahbdt2cpdHRCQbqwzdIgF74jhsrl4xGKML2A7QNG7C\ngJWBIAjo128AOnfuioUL5+Grr75Anz7Pol+/AZg7dz6qVasmd4lERGZn8aGrD9gTx/NnsKUFbOOm\nupObGLCyc3f3wHvvLcWAAYMwbdpkbNnyLX755Se8+eYcDBkyDOoqegIaEVVNFhW6QlJi0V3EJQRs\nTpOm0DRqwoC1Ak2bNsdPP/2Br79ejQUL3sEbb0zGpk3r8P77y9GoURO5yyMiMgvZQteogPXwQHa7\nJ3Nnr00YsFZOrVZjxIhX8OyzPTFnzkxs27YFXbp0wPDhIzF9+iy4WfEFQ4iIjGGW0DUI2JPHYXvy\nONTXrhqMYcBWHX5+/li16ku88MKLmD59Cr744lPs2rUD8+YtRK9effnZXiJSrMoJ3T/+gOPev2Bz\n6oRxAdu4KcTAIAZsFdO+/ZPYu/cwVq5cjuXLl2DUqOFYv34tFi1agtDQcLnLIyIyucoJ3U6dkHcR\nQIOAzTsGy4ClXPb29pgy5Q306dMfM2ZMxR9//Ib27R/HuHGTMGHCFDgo9DKaRFQ1VU7oTp+OpPD6\nDFgyWu3aIdi4cSu+/34XZs16A0uXLsLWrd/lnvncW+7yiIhMotS12GJjYzFkyBA888wz6NGjB9as\nWVP6VhcuRHaPXjwmS2UiCAJ69HgOf/31D0aNeg03bsRg4MA+6NevH44d+wdSoWthExFZm1JDV61W\nY8aMGfjxxx+xadMmrF+/HpcvXy7ty4jKzcXFFfPmLcSvv+5HixaPYuvWrXj66U7o0OFxfPbZx0hI\nKLo+MRGRNSg1dH19fVGvXj0AgLOzM0JDQ3Hv3r1KL4yoQYOG+P77X/DTTz+hZ8/euHQpGrNmTUej\nRhEYNWoY9u/fC1EU5S6TiMhoZTqme/PmTVy4cAGNGjWqrHqIDKhUKnTt2hXNmrVGfHw8Nm/ehHXr\nvsb27VuxfftWBAUFIzJyCAYOjIS/f4Dc5RIRlUiQjDxQlpaWhhdffBGvvvoqnnrqqRLHBgej2BnI\nsWNpxY5v3ty52NflHK9SqYr0YE315ynYhyXUU57xeT3kjZckCX//fRTr13+DXbu2Iz39LADAwcER\nLi4ucHBwgCAIFlN/npgYFeKKWbnK0v/8C4/39XU16EPuesozvmAPllBPecf7+roiMLD4vT3WUD8A\ntGzpavV5Aej+fhvDqJmuRqPB+PHj8dxzz5UauHlUqqIF+Pq6PmRs8duQe3zhHuSup7zj8/qwlHrK\nM16lUhmMf/bZznj22c5ISkpCSIgKqakpyMzMQGZmBtRqNVxcXJCUdB9hYWEWUX9JX2MNf/6Fxxd8\nbAn1lGd83nNLqaf844v/AmupX/c11p8XxjJqpjtt2jR4enpixowZRm+4uP/RW5PC/5u3Vkrow9ge\nTp8+hQ0b1mDLlu+QlJQIAGjbth0iI4ege/eesn/mVwk/C0AZfSihB0AZfSihB6Dk/1QUVGpGHzt2\nDLt378aRI0fQq1cv9O7dG/v3769wgUSm1rBhIyxcuASnTkXh448/R5s2T+Dgwf0YM+ZlNGpUBzNn\nvo6zZ8/IXSYRVWFGH9MtK2v/n4uS/vdl7X1UpIcrVy5hw4Z12LhxHeLidGfdN2vWHJGRL6F3775w\ncTHuf6emoISfBaCMPpTQA6CMPpTQA2DCmS6RNQsJCcOsWXNx4sR5fPPNRnTp0g0nThzHlCnj0aBB\nHUyc+Br++ecoL7xBRGbB0KUqwdbWFk8/3R3r1n2H48fPYcaM2fDx8cWGDWvRvXtntGvXCqtWrcT9\n+7zwBhFVHoYuVTkBAdUxadLr+PvvE9i8eSd69eqDq1ev4K23ZqJRozoYOXIo9u79gxfeICKTk20R\neyK5qVQqtG//JNq3fxL379/Hli2bsH79GuzcuQ07d25DYGAQXnhhMF54YTCqV68hd7lEVMmys4H0\ndCA9XShwr3uclpb/WkZG0TEbNxr3PXgi1UMo6eC+tfdhzh4kScKxY/9g/fo12L59K9LT06BSqdCx\n41OIjHwJXbp0g62tbbm2rYSfBaCMPpTQA6CMPsrSgyiimMDLv8/IKDksC79XOEA1mvIv0GNsknKm\nS1SAIAho0eJRtGjxKObNW4gdO7Zh/fpv8Ntvv+C3336Br281DBgwCIMHD0FISNELbxCRjiQBaWlA\naqqAlBQBKSmGj9PSdI+1WiA+3v6hQVg4LE1BpZLg5AQ4OenuvbzEAs91rzk7S3B0zB9jeF/0NehX\nkS8ZZ7oPoYT/QQLK6MMSejh37iw2bFiDzZs34cGDBwCA1q3bIjJyCJ599jk4OjqWug1L6MMUlNCH\nEnoATN+HJAFZWXnhmB+SqanIDUvd4/zwzH+vuK+RpPKHpL19ySFX8N7R0XCMs/PDw9LREbC3N/2q\ns8Z+ZIih+xD8S2k5LKmHzMxM7NnzPdatW4MDB/YCANzc3NGv3/OIjHwJDRs+fDEQS+qjIpTQhxJ6\nAPL70GhQKAx1j4ubZRYOybzHea/n5JQvjezsJLi6SnBxAVxcdI9dXQFXVwnOzvmPdWN0z11cJNSs\n6YTs7LQiM0y12sR/WJWMoVtBSvtLac0stYdr165i48a12LhxPWJj7wAAGjduisjIIejTpx/c3NwN\nxltqH2WlhD4srQdJ0h2rTEwUkJgoICkp7x548CD/eeH309JUSE6Wyr3bVRAMw9DZuWAwokBA5j/P\nC1NdkOaHp719+Xq3tJ9FeTF0K0hJvwjW3oel96DRaPD7779i/fpv8OuvP0Or1cLR0RE9e/ZGZORL\naNXqMQiCYPF9GEsJfVRWD5mZKBSQMAjJwqFZ8P3sbOOD09ZWgru7BC8vFRwdtUVmjwXDMO/1ggGa\n956Tk+l3s5aVEn6fAIZuhSnpF8Ha+7CmHmJj7+Dbbzdg/fo1uHbtKgAgLCwckZEvYeTIobCzc5O5\nwoqzpp/Hw5TUQ04ODELz4YFZ9P3MTOMTTK2W4OEhwd0d8PCQ9Dd3d6nQ86Lv54Wl0n8W1oShW0FK\n+kWw9j6ssQdRFHHo0EGsW/cNfvhhF7KysgAAoaFhaN26rf4WEFBd5krLzlp+Hnlnz8bHC7h/X0B8\nvID4eBXu3xeQnm6PO3dy9KFZcBduWXbVCoIuFN3dJXh65gem4fOi73t46HbXVnSWaS0/i5IooQeA\noVthSvpFsPY+rL2HBw8SsG3bZhw48Cf27z+A1NT8XkJCQvUB3KbNE1YRwnL+PDIzDUM0Li7vsapQ\nuOoeZ2QYl2pubg+bZepC82GzUFfXsq+nakrW/ncDUEYPAEO3wpT0i2DtfSihB0DXx507D3DmzCn8\n9ddBHDp0AEeOHEZKSrJ+TO3aIWjT5gk8/ngbtGnzhEVeCcuUP4+cHCAhQReexYWmLlhV+sepqaWH\nqL29BB8fw5u3twQfH1H/PDTUCZKUCg8PCW5ugI2VXrFACX83lNADwNCtMCX9Ilh7H0roASi+D61W\naxDChw8fMgjh4ODaBiFco0ZNc5ddREk/D61Wd7Zt4QDNn5EWfE+FxMTSQ9TGJi808wPU17domOa9\n7uxc+m5bJf9OWRsl9AAwdCtMSb8I1t6HEnoAjOtDq9Xi7NnTBiGcnJykfz8oKNgghGvWrFXZZUOj\nAe7dE3DnjoDYWBUyMx1x7VpWkRlpfLyAhAQBolhy4qlUEry8Cs9Ciz7OC1N398q5kEFV+Z2ydEro\nAWDoVpiSfhGsvQ8l9ACUrw+tVotz587gr78O4NChgzh8+BCSkhL17wcGBqNNm/wTs2rVCizT9tPS\ngNhYAbdvq/SheueOgNu38x/fu1d6kHp46EKy5Bmp7ubpKcl+4YOq/DtlaZTQA8DQrTAl/SJYex9K\n6AEwTR+6ED6LQ4cO4K+/DuLw4b8KhXAQWrdui8cea4v69dtDrQ7MDVEVYmMF3LmjC1LdTYXk5IeH\nqZ2dBH9/CQEBIgICJAQESPD3FxEW5gBb23T4+OhC1ctLQjnXgJANf6cshxJ6AIwPXSs9fYCoalKr\n1ahTpxHc3BqjcePx6NVLwokT93DqVDyuXMnErVu22LTJD5s21QBg99DtuLtLqFFDRPPmulD195dQ\nvXr+44AA3ey0uN26vr4OiIvTVl6TRArG0CWyEJIEJCejwK5e3Wy04K7e2FjdCUiGaufedMdLfXyy\n4eAQj5yca0hMPI2srCsAbgG4CT8/EW3ahKB9+1Zo3botAgODIMh9SSKiKoShS2QGWi1w6xZw5oyq\nwK7egrt7da+VdGEGJyfdDLRuXU3uzFTM3eWrm6FWr67b3as7XuoKoCFE8RGcP38Ohw8fxF9/peDw\n4YPYtu0Atm37BgBQo0ZN/WeEW7dui6CgYIYwUSXiMd2HUNJxBmvvw1p6SEkBrl9X4do1Fa5fF3D9\nukr//ObNkldv8fExPG4aEKAL1bxdvQEBItzcKn4WryiKuHDhfG4IH8Thwwdx//59/fs1atTUnxnd\nunVbBAfXLhLC1vLzKIkSegCU0YcSegB4IlWFKekXwdr7sJQetFrdmb7Fher16wLu3y/+0kQ+PiKC\ngiSEhqrh5ZVtcGJSQIAIP7/yr9BSUaIoIirqAg4dOoBDh/7CoUMHDEK4evUaBiFcu3YIqlVzs4if\nR0VYyu9URSmhDyX0ADB0K0xJvwjW3oc5e0hNhT5M84JVF6oq3LhR/EowtrYSAgMlBAWJ+ltwcP5z\nFxfz91FekiQhKuoC/vrrAA4f1oVwfHy8/n1//wA0bdoEgYEhqFMnAuHhEQgPrwNvb28Zqy47a/hZ\nGEMJfSihB4BnLxMVSxR1s9W8UL12LT9Ur18v7iQlHW9vEQ0aFAxV3ew1KEg3a5X7c6emIggC6tat\nh7p162HEiFcgSRIuXozSh/CRI4ewZ8+eIl/n7e2tD+D8WwRq1qwFlZwXJyayMAxdUpz0dBjMVAvu\nAo6JUSErq+hs1cZGQq1aEho00BQJ1aAg3fHUqkgQBERE1EVERF0MHz4SAGBjo8GRI/8hOvoiLl6M\nwqVLuvu//z6CI0cOGXy9o6MjQkPDUadOnQKhHIGQkFDYy7VPnUhGDF2yOpIE3L1reGy14Gz13r3i\nZ1aenhLq1Ss6Uw0K0p35a60XvTc3T09PtGjxKFq0eNTg9czMTFy9egXR0VEFwvgiLl+OxpkzpwzG\nqlQqBAUFo06dCISF1cndVa2bIbu7e5izHSKz4j8zZJEkSbcb+MIFFWJjgTNn7PWhGhOjKnbJNrVa\nQs2aEtq10+hDVXevu7m7y9BIFeLg4IB69eqjXr36Bq+LooibN2/khvFF/cw4OjoKP/+8Bz//bLi7\nulo1v9wwDjc4bhwQUJ0fZyKrx9AlWUmS7mL6Fy6oEBWlu124oMbFiyokJRX8B1Z3dSU3Nwnh4WKB\nMJX0M9caNThbtUQqlQqBgUEIDAxCp05dDN67f/8+oqOj9Luqo6OjcOlSNA4e3I+DB/cbjHV2dkF4\neDjCwyMMZsjBwbVha23XoaQqi/9EkdnExeWHa37Iqoss76ZWSwgJEfHEEyIiIkS0bGkPb+80BAWJ\n8OCeR0Xx9vaGt3drPPZYa4PX09PTcflydIEw1s2Qz507ixMnjhuMtbGxQe3aIQXCOFx/7+Ji3Bml\nRObC0CWTu39fKBSsulvhz7GqVBJq15bQurUGdevqAjYiQkRoqGjwuVVfX3vExYlm7oLk5OTkhIYN\nG6Nhw8YGr2s0GsTEXEN0dLTBSVzR0RcRHX0RP/6422B89eo1DM6mzpsh+/i4mLMdIj2GLpXbgwdA\nVJS60K5hVZGP3QiChKAgCS1b5hiEa1iYCAcHmYonq2RjY4OQkDCEhISha9en9a9LkoR79+4VOYkr\nOjoK+/b9iX37/jTYjouLC/z9A+DvHwA/P//cez+D1/z8/OHk5GTuFknhGLpUqqQk4MIFtUGwRkWp\nij1LODBQRJcuGkREaBERIaJuXV248t8uqkyCIMDPzw9+fn5o27adwXupqSkFPt6kmyHfvHkdt2/f\nxqVL0SVu193dA/7+/vDzC4C/v39uKPvnhnL+Y378iYzF0CW9lBTkBqraIFxjY4uGa61aIp56SpM7\na9Wibl0R4eEinJ1lKJyoBC4urmjatDmaNm2ufy3vKkhZWVm4d+8u7t6NRWxsLO7evYPY2FjExt5B\nbOyd3NfvICrqQonfw8vLq5hgDjAI6WrV/HjCFzF0q6LUVODixfwzhfPC9fbtouFao4aIjh01ubNW\n3ey1Tp38SxsSWTN7e3vUqhWIWrUCSxyXkZGBe/fuFgjm/HDOC+Zbt27i/PmzD92GIAjw9vbRB3HB\nXdsFw9nHxxc2PA1fsfiTVbD0dODff4HDh230s9eoKBVu3CgargEBIjp00Oh3CeftHnblyZ9EcHR0\nRFBQMIKCgkscl5aWhrt3Y/VBnB/M+Y+vXLlc5GIhBalUKvj6Vis0Yy46g7a2612TDkNXITQa3a7h\n48fVOH5chf/+081gRREAHPXjqlUT8cQT+WcL581eeeEIoopzdnZGSEgoQkJCSxyXmppisBu74K7t\n/F3a53Hy5PGHbsPGxgZeXl5wc3OHu7s73N09Ctx76J97eHjAza3ovVopFwy3MgxdKyRJQEyMgOPH\n1fjvP13InjqlNrhKk5OThJYttWjZ0gaBgZn62aunp4yFExEA3XHmsDBXhIWFP3SMJElITk4q9hjz\n3bt3ERt7B8nJiUhIeICYmOvIzs4uUw2urm7FhLW7QVjnv+ZpEOCOjo68Olg5MXStQEICcOJEXsDq\nQrbgx3JUKgl164po1kyLZs1ENG2qm73a2OSdMJIjY/VEVB6CIOhnrBERdYsdk3dCmCRJyMjIQHJy\nEhITE5GUlISkpAe597rniYmJ+vcL3sfEXEdKSnKZarOzs9PPmjnLLhuGroXJyABOn87bTawL2mvX\nDI/BBgaKeO65HDRtqgvZhg21PGuYqAoTBAFOTk5wcnKCv39Amb9eq9UiOTnJIKSTkhILBHhigZth\nkF+/fg05OWX7j72rq5s+gH18vGBraw9HR139jo6OcHJy1t87ORV87lRgXP69s7Pu3hrCnKErI60W\niI5W4b//VPpZ7PnzKmg0+bttPD0ldOyoyQ1YLZo0EeHrK8lYNREpjVqthqenFzw9vcr8tWWdZRcM\n7piY6zh79rTJ+rC3ty8S2oXDOu/2sJA3DHvDcLezs6vwbnWGrplIEnD7tqA/Bnv8uBonTqiRlpb/\nA7S3l9CkiW43cdOmulvt2hJ46ISILFVFZ9ne3s6IibmH9PR0ZGSkF3ufd8vIyEB6elqh+4LjdK+l\npaUjOTkZsbGxyMhIhyia5jKyarW6SFjnzcT3799r1DZKDd2ZM2di79698Pb2xu7du0sbTrmSkqDf\nRZx3NnHBKzgJgoSICBFNm4r6WWzduiLs7GQsmojIzFQqFZydneFcScfIJElCVlZWgSDXBXZ6enEB\nXtmry4cAAAsRSURBVFyQFwx9w/vExESkp6eVafd6qaHbp08fvPjii5g2bVqFGleyrCzg7FmVwdnE\nly4ZHluoXl1E9+45aNpUN5Nt3FjLz8ASEVUyQRDg4OAABweHcu0+N4ZJQ7dFixa4detWhQpSElEE\nLl/WHYfNm8meOaNCTk7+PmBXV91C6rrdxLqZrL8/j8MSESlRWS7vyWO6pbh7V3ccNu9kpxMn1EhJ\nyQ9YW1sJDRqI+mOwzZrplqZTFb3oExERVXEM3QIkCTh/XoUDB9Q4fhw4csS5yPWIw8K06NYt/2Sn\nRx4xXPuViIjoYSotdH19reOA5Y0bwG+/6W6//w7cvZv/np+fCj17Ao8+CrRqBbRoAXh4qAGoAVjP\naiHW8rMoiRJ6ANiHJVFCD4Ay+lBCD8YyKnQlqezHI+PiUsr8NeaQlAQcPGiD/fvV2L/fBpcv589k\nq1UT0a+fFu3aadCzpyMcHVMMPq6TkwPExclQdAXkXbHGmimhB4B9WBIl9AAoow8l9AAY/x+HUkN3\nypQpOHr0KBITE9GhQweMGzcOffv2rXCB5pKVBfzzj1ofsidOqCCKuiR1dpbQpYsG7dpp0K6d7tKJ\neSHr62t9AUtERJat1NBdunSpOeowGVHUfXxn3z5dyB49mr8QgI2NbhGAdu10t2bNtOCa0kREZC6K\nOJHq+nUB+/frdhkfOKBGQkL+LuN69fJCVoPHH9dy8XUiIpKNVYZuQoLuuGzebPb69fyQrV5dxMCB\nOWjXToMnntDCz4+fjyUiIstgFaGbkQEcPZp/XPb0aRUkSbfL2M1NwjPP5Ohns6GhvFYxERFZJosM\nXa0WOHVKpd9l/PffamRl6ZLUzk5Cmzb5u4wbNdKtG0tERGTpLCKuJAm4elXAvn26kD140AZJSfnT\n1YYN80O2VSstnJxkLJaIiKicZAvde/cEHDyYv8v45s3847KBgSJ69tTtMm7TRgsfHx6XJSIi62e2\n0E1NBY4cUetns+fP56/C4+kp6UO2XTsNgoMZskREpDyVFro5OcDx4/nHZf/9Vw2NRrfL2MFBQvv2\nugtStG+vQYMGXCCAiIiUr1JCt2dP4M8/XZCaqgtZQZDQpImov/JTy5ZaODhUxncmIiKyXJUSurt3\nAyEhEvr10+0ybttWAw+PyvhORERE1qNSQvfaNcDJKa0yNk1ERGS1KuVIalBQZWyViIjIuvH0JSIi\nIjNh6BIREZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIR\nEZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eI\niMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMGLpE\nRERmwtAlIiIyE4YuERGRmRgVuvv370e3bt3QtWtXfPbZZ5VdExERkSKVGrqiKGLevHlYvXo1vv/+\ne/zwww+4fPmyOWojIiJSlFJD99SpUwgKCkKNGjVga2uL7t274/fffzdHbURERIpSaujevXsXAQEB\n+ud+fn64d+9epRZFRESkRKWGriRJ5qiDiIhI8WxKG+Dv74/bt2/rn9+9exfVqlUrdcO+vq4Vq8wC\nKKEHQBl9KKEHgH1YEiX0ACijDyX0YKxSZ7oNGzZETEwMbt26hezsbPzwww/o1KmTOWojIiJSlFJn\numq1GrNnz8bw4cMhSRL69euH0NBQc9RGRESkKILEg7ZERERmwStSERERmQlDl4iIyEwYukRERGZS\n6olUZbF//34sWLAAkiShb9++eOWVV0y5ebOYOXMm9u7dC29vb+zevVvucsolNjYW06ZNQ3x8PNRq\nNfr3748hQ4bIXVaZZWdnIzIyEjk5OdBqtejatSvGjh0rd1nlIooi+vbtCz8/P6xatUrucsqlY8eO\ncHFxgUqlgo2NDbZs2SJ3SeWSkpKCN998E9HR0VCpVFiwYAEaN24sd1lGu3r1KiZNmgRBECBJEm7c\nuIEJEyZY5d/xr7/+Glu2bIEgCKhTpw4W/r+9u3mJag8DOP6dHKRQexElCyzIjCySFr1AEyamSTXV\nxGCLNiVRbdIow14oghYJLfoHWkREEBEaRG1EszGmQiuGYIgwIhhMKkRT5yXPnOcu4l64G+89x7nz\na7rPZz1n+A6HmYcznHmmo4P8/HzTWY7cunXrr/fCv/qslQxJp9NSX18vsVhMfvz4IXv37pWhoaFM\nPX3WDAwMSDQaFb/fbzrFtS9fvkg0GhURkcnJSdmxY0dOngsRkXg8LiIilmVJU1OTRCIRw0Xu3Lx5\nU9ra2uT48eOmU1yrq6uTsbEx0xmzdvbsWbl//76IiExPT8vExIThIvfS6bT4fD4ZHh42neLYyMiI\n1NXVSSqVEhGRkydPSldXl+EqZ96/fy9+v19SqZRYliWHDx+WT58+zXhMxr5e/l12NG/YsIH58+eb\nzpiV0tJSqqqqACgoKKCioiJnV3fOmzcP+HnVa1mW4Rp3RkZGePr0KU1NTaZTZkVEsG3bdMasTE5O\nMjg4SDAYBMDr9VJYWGi4yr1wOMyyZcv+tqo3l9i2TSKRwLIsksnkv1q89Cv58OED69evJz8/n7y8\nPDZu3Eh3d/eMx2Rs6OqO5l9TLBbj3bt3VFdXm05xxbZtAoEAPp8Pn8+Xk6/j6tWrtLe34/F4TKfM\nisfj4ciRIwSDQe7du2c6x5VYLMaiRYs4f/48+/fv59KlSySTSdNZrj1+/Jjdu3ebznBl8eLFNDc3\nU1tbS01NDUVFRWzZssV0liOVlZUMDAwwPj5OIpEgFArx+fPnGY/J2NAV/bnvL2dqaorW1lYuXLhA\nQUGB6RxX5syZw4MHDwiFQkQiEYaGhkwnOdLX10dJSQlVVVU5/x65e/cunZ2d3Lhxgzt37jA4OGg6\nyTHLsohGoxw8eJCuri7mzp2bs/8RPj09TW9vLzt37jSd4sr379/p6enhyZMn9Pf3E4/Hc+4+moqK\nCo4ePUpzczPHjh1j9erVeL0z3yqVsaHrdkez+m9YlkVrayv79u2jvr7edM6sFRYWsmnTJvr7+02n\nOPL69Wt6e3vZvn07bW1tvHz5kvb2dtNZrpSWlgJQXFxMQ0MDb9++NVzkXFlZGWVlZaxbtw6AxsZG\notGo4Sp3QqEQa9eupbi42HSKK+FwmPLychYuXEheXh4NDQ28efPGdJZjwWCQzs5Obt++zYIFC1i+\nfPmMj8/Y0P2ddjTn+hUJ/LwLe+XKlRw6dMh0imujo6NMTEwAkEwmef78OStWrDBc5czp06fp6+uj\np6eH69evs3nzZq5du2Y6y7FEIsHU1BQA8XicZ8+eUVlZabjKuZKSEpYsWcLHjx8BePHiRc6utX30\n6BF+v990hmtLly4lEomQSqUQkZw9F6OjowAMDw/T3d39j+ckYz8Z+l12NP95NTI2NkZtbS0tLS1/\n3XSRK169esXDhw9ZtWoVgUAAj8fDqVOnqKmpMZ3myNevXzl37hy2bWPbNrt27WLbtm2ms/6Xvn37\nxokTJ/B4PKTTafbs2cPWrVtNZ7ly8eJFzpw5g2VZlJeX09HRYTrJsWQySTgc5sqVK6ZTXKuurqax\nsZFAIIDX62XNmjUcOHDAdJZjLS0tjI+P4/V6uXz5MkVFM/9jku5eVkoppbJEN1IppZRSWaJDVyml\nlMoSHbpKKaVUlujQVUoppbJEh65SSimVJTp0lVJKqSzRoauUUkpliQ5dpZRSKkv+AO2e4yf8wTuC\nAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0xc1dc310\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "# Train our variables.\n", + "\n", + "# numpy is used for its asscalar() function.\n", + "import numpy as np\n", + "\n", + "num_training_steps = 10\n", + "\n", + "def train_model(inputs, labels, wb, optimizer, num_training_steps):\n", + " loss_at_step = []\n", + " w_at_step = []\n", + " b_at_step = []\n", + " for step_num in range(num_training_steps):\n", + " loss, gradients_and_variables = value_and_gradients_fn(inputs, labels, wb)\n", + " loss_at_step.append(np.asscalar(loss.numpy()))\n", + " \n", + " optimizer.apply_gradients(gradients_and_variables)\n", + " w, b = wb.variables\n", + " w_at_step.append(np.asscalar(w.read_value().numpy()))\n", + " b_at_step.append(np.asscalar(b.read_value().numpy()))\n", + "\n", + " print(w_at_step)\n", + " t = range(0, num_training_steps)\n", + " plt.plot(t, loss_at_step, 'k',\n", + " t, w_at_step, 'r',\n", + " t, [true_w] * num_training_steps, 'r--',\n", + " t, b_at_step, 'b',\n", + " t, [true_b] * num_training_steps, 'b--')\n", + " plt.legend(['loss', 'w estimate', 'w true', 'b estimate', 'b true'])\n", + " plt.show()\n", + "\n", + "train_model(inputs, labels, wb, optimizer, num_training_steps)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "UNurY9VJ-hpH" + }, + "source": [ + "## Other Ways to Compute Gradients\n", + "\n", + "Using our loss function as an example (`calculate_linear_model_loss()`), there are several other ways we could compute gradients:\n", + "\n", + "1. `tfe.implicit_gradients()`\n", + "1. `tfe.gradients_function()`\n", + "1. `tfe.implicit_value_and_gradients()`\n", + "1. `tfe.value_and_gradients_function()`\n", + "\n", + "Each of these functions does the following:\n", + "* Wraps a function.\n", + "* Returns a function with the same input signature as the wrapped function.\n", + "\n", + "They differ only in what information they return.\n", + "\n", + "### Gradients-only functions\n", + "\n", + "The following two functions return a function that returns only the variables' gradients:\n", + "\n", + "1. `tfe.gradients_function()`: Returns the partial derivatives of the function `f()` with respect to the parameters of `f()`.\n", + "1. `tfe.implicit_gradients()`: Returns the partial derivatives of the function `f()` with respect to the trainable parameters (`tf.Variable`) used by `f()`.\n", + "\n", + "In our example above, the `tf.layers.Dense` object encapsulates the trainable parameters.\n", + "\n", + "### Value and gradients functions\n", + "\n", + "The following two functions are identical to their counterparts above, except that they also return the value of the wrapped function.\n", + "\n", + "1. `tfe.implicit_value_and_gradients()`\n", + "1. `tfe.value_and_gradients_function()`\n", + "\n", + "### Gradient demos\n", + "\n", + "In the demos below, we show examples for the `implicit_*` functions, since our existing loss function works seamlessly with these versions. (The other versions require that your parameters are tensors and tensors only; in our example, we're using a `Dense` layer.)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 85, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 100, + "status": "ok", + "timestamp": 1505502831671, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "aEoCftnfAIH5", + "outputId": "72f1c1dc-a574-463f-f860-c4e5f48fcdaa" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[(\u003ctf.Tensor: id=673, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", + " (\u003ctf.Tensor: id=671, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)]" + ] + }, + "execution_count": 13, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# tfe.implicit_gradients() demo\n", + "gradients_fn = tfe.implicit_gradients(loss_fn)\n", + "\n", + "# Returns only gradients and variables:\n", + "gradients_fn(inputs, labels, wb)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 102, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 88, + "status": "ok", + "timestamp": 1505502831785, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "bbgCUdCzAVhH", + "outputId": "152aa9b6-9e42-4b7e-848a-9423c0b1929c" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(\u003ctf.Tensor: id=688, shape=(), dtype=float32, numpy=1.0623235\u003e,\n", + " [(\u003ctf.Tensor: id=720, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", + " (\u003ctf.Tensor: id=718, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)])" + ] + }, + "execution_count": 14, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# tfe.implicit_value_and_gradients() demo\n", + "value_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)\n", + "\n", + "# Returns only gradients:\n", + "value_gradients_fn(inputs, labels, wb)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Eager Execution Tutorial: Working with Gradients", + "provenance": [], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb new file mode 100644 index 0000000000..ff0ff4a6a7 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U9i2Dsh-ziXr" + }, + "source": [ + "# Eager Execution Tutorial: Importing Data\n", + "\n", + "This notebook demonstrates the use of the [`tf.contrib.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n", + "\n", + "* Creating a `Dataset`.\n", + "* Iteration over a `Dataset` with eager execution enabled.\n", + "\n", + "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n", + "\n", + "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly different. You will use a Pythonic `Iterator()` class instead of using `make_one_shot_iterator()` and `get_next()`. As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z1JcS5iBXMRO" + }, + "source": [ + "# Setup: Enable eager execution\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RlIWhyeLoYnG" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "from tensorflow.contrib.eager.python import tfe\n", + "\n", + "# Enable eager execution\n", + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H9UySOPLXdaw" + }, + "source": [ + "# Step 1: Create a source `Dataset`\n", + "\n", + "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "WPTUfGq6kJ5w" + }, + "outputs": [], + "source": [ + "ds_tensors = tf.contrib.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", + "\n", + "# Create a CSV file\n", + "import tempfile\n", + "_, filename = tempfile.mkstemp()\n", + "with open(filename, 'w') as f:\n", + " f.write(\"\"\"Line 1\n", + "Line 2\n", + "Line 3\n", + " \"\"\")\n", + "ds_file = tf.contrib.data.TextLineDataset(filename)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "twBfWd5xyu_d" + }, + "source": [ + "# Step 2: Apply transformations\n", + "\n", + "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.contrib.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset) for details." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ngUe237Wt48W" + }, + "outputs": [], + "source": [ + "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", + "ds_file = ds_file.batch(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IDY4WsYRhP81" + }, + "source": [ + "# Step 3: Iterate\n", + "\n", + "Use `tfe.Iterator` on the `Dataset` object to get a Python iterator over the contents of the dataset.\n", + "\n", + "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that this process of iteration is different. Here there are no calls to `Dataset.make_one_shot_iterator()` and no `get_next()` calls." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 153, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 201, + "status": "ok", + "timestamp": 1505952405928, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "lCUWzso6mbqR", + "outputId": "ec027d30-96c6-4ea4-9ee1-ef74ec1ae29a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Elements of ds_tensors:\n", + "tf.Tensor([4 9], shape=(2,), dtype=int32)\n", + "tf.Tensor([16 25], shape=(2,), dtype=int32)\n", + "tf.Tensor([36 1], shape=(2,), dtype=int32)\n", + "\n", + "Elements in ds_file:\n", + "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n", + "tf.Tensor(['Line 3' ' '], shape=(2,), dtype=string)\n" + ] + } + ], + "source": [ + "print('Elements of ds_tensors:')\n", + "for x in tfe.Iterator(ds_tensors):\n", + " print(x)\n", + "\n", + "print('\\nElements in ds_file:')\n", + "for x in tfe.Iterator(ds_file):\n", + " print(x)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Eager Execution Tutorial: Importing Data", + "provenance": [], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py new file mode 100644 index 0000000000..7a213e9e03 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py @@ -0,0 +1,51 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.eager.python.examples import cart_pole_helper +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +class RewardDiscountingTest(test_util.TensorFlowTestCase): + + def testDiscountingRewards(self): + rewards = [0.0, 10.0, 20.0] + discount_rate = 0.9 + self.assertAllClose( + [10 * discount_rate + 20 * discount_rate * discount_rate, + 10 + 20 * discount_rate, 20], + cart_pole_helper.discount_rewards(rewards, discount_rate)) + self.assertAllClose( + [-1.2], cart_pole_helper.discount_rewards([-1.2], discount_rate)) + self.assertEqual([], cart_pole_helper.discount_rewards([], discount_rate)) + + def testDiscountAndNormalizeRewardSequences(self): + rewards1 = [0.0, 10.0, 20.0] + rewards2 = [0.0, 5.0, -5.0] + reward_sequences = [rewards1, rewards2] + discount_rate = 0.9 + dn = cart_pole_helper.discount_and_normalize_rewards(reward_sequences, + discount_rate) + self.assertAllClose( + [[1.03494653, 1.24685514, 0.64140196], + [-0.83817424, -0.83439016, -1.25063922]], dn) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py new file mode 100644 index 0000000000..dc1381cc04 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py @@ -0,0 +1,162 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit test for cart-pole reinforcement learning under eager exection.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import glob +import os +import shutil +import tempfile +import time + +import gym +import numpy as np + +from tensorflow.contrib.eager.python.examples import cart_pole +from tensorflow.python.eager import context +from tensorflow.python.eager import test +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import random_seed +from tensorflow.python.framework import test_util +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.training import training + + +class CartPoleTest(test_util.TensorFlowTestCase): + + def setUp(self): + super(CartPoleTest, self).setUp() + self._tmp_logdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tmp_logdir) + super(CartPoleTest, self).tearDown() + + def testGetLogitsAndAction(self): + hidden_size = 5 + policy_network = cart_pole.PolicyNetwork(hidden_size) + + dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5], + [0.0, -0.2, 0.6, -0.8]], dtype=np.float32) + logits, actions = policy_network.forward(constant_op.constant(dummy_inputs)) + + self.assertEqual((2, 1), logits.shape) + self.assertEqual(dtypes.float32, logits.dtype) + self.assertEqual((2, 1), actions.shape) + self.assertEqual(dtypes.int64, actions.dtype) + + def testCrossEntropy(self): + hidden_size = 5 + policy_network = cart_pole.PolicyNetwork(hidden_size) + + dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5], + [0.0, -0.2, 0.6, -0.8]], dtype=np.float32) + cross_entropy = policy_network._get_cross_entropy_and_save_actions( + constant_op.constant(dummy_inputs)) + + self.assertEqual((2, 1), cross_entropy.shape) + self.assertEqual(dtypes.float32, cross_entropy.dtype) + + def testPlayAGame(self): + hidden_size = 5 + cart_pole_env = gym.make("CartPole-v0") + cart_pole_env.seed(0) + cart_pole_env.reset() + + device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" + logging.info("device = %s", device) + with context.device(device): + policy_network = cart_pole.PolicyNetwork(hidden_size) + policy_network.play(cart_pole_env, max_steps=10, render=False) + + def testTrain(self): + hidden_size = 5 + num_games_per_iteration = 5 + max_steps_per_game = 10 + discount_rate = 0.95 + learning_rate = 0.02 + + cart_pole_env = gym.make("CartPole-v0") + cart_pole_env.reset() + + device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" + logging.info("device = %s", device) + with context.device(device): + policy_network = cart_pole.PolicyNetwork(hidden_size, + train_logdir=self._tmp_logdir) + optimizer = training.AdamOptimizer(learning_rate) + policy_network.train( + cart_pole_env, + optimizer, + discount_rate, + num_games_per_iteration, + max_steps_per_game) + self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) + + +class EagerCartPoleTrainingBenchmark(test.Benchmark): + + def benchmarkEagerCartPolePolicyNetworkTraining(self): + burn_in_iterations = 1 + benchmark_iterations = 2 + num_games_per_iteration = 10 + max_steps_per_game = 100 + discount_rate = 0.95 + learning_rate = 0.02 + + cart_pole_env = gym.make("CartPole-v0") + cart_pole_env.seed(0) + random_seed.set_random_seed(0) + cart_pole_env.reset() + + hidden_size = 5 + policy_network = cart_pole.PolicyNetwork(hidden_size) + optimizer = training.AdamOptimizer(learning_rate) + + # Perform burn-in. + for _ in xrange(burn_in_iterations): + policy_network.train( + cart_pole_env, + optimizer, + discount_rate, + num_games_per_iteration, + max_steps_per_game) + + gc.collect() + start_time = time.time() + for _ in xrange(benchmark_iterations): + policy_network.train( + cart_pole_env, + optimizer, + discount_rate, + num_games_per_iteration, + max_steps_per_game) + wall_time = time.time() - start_time + # Named "examples"_per_sec to conform with other benchmarks. + extras = {"examples_per_sec": benchmark_iterations / wall_time} + self.report_benchmark( + name="EagerCartPoleReinforcementLearning", + iters=benchmark_iterations, + wall_time=wall_time, + extras=extras) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py new file mode 100644 index 0000000000..aee0b3d0dd --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py @@ -0,0 +1,114 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit tests for linear regression example under TensorFlow eager execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import glob +import os +import shutil +import tempfile +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python.examples import linear_regression +from tensorflow.python.eager import context +from tensorflow.python.eager import test +from tensorflow.python.framework import test_util +from tensorflow.python.platform import tf_logging as logging + + +def _create_data_gen_for_test(): + true_w = np.array([[1.0], [-0.5], [2.0]], dtype=np.float32) + true_b = np.array([1.0], dtype=np.float32) + noise_level = 0 + batch_size = 64 + return ( + true_w, true_b, noise_level, batch_size, + linear_regression.DataGenerator(true_w, true_b, noise_level, batch_size)) + + +class LinearRegressionTest(test_util.TensorFlowTestCase): + + def setUp(self): + super(LinearRegressionTest, self).setUp() + self._tmp_logdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tmp_logdir) + super(LinearRegressionTest, self).tearDown() + + def testSyntheticBatch(self): + _, _, _, batch_size, data_gen = _create_data_gen_for_test() + + xs, ys = data_gen.next_batch() + self.assertEqual((batch_size, 3), xs.shape) + self.assertEqual((batch_size, 1), ys.shape) + self.assertEqual(tf.float32, xs.dtype) + self.assertEqual(tf.float32, ys.dtype) + + def testLinearRegression(self): + true_w, true_b, _, _, data_gen = _create_data_gen_for_test() + + learning_rate = 0.1 + num_iters = 40 + + device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" + logging.info("device = %s", device) + with context.device(device): + linear_model = linear_regression.LinearModel() + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + linear_model.fit(data_gen.next_batch, optimizer, num_iters, + logdir=self._tmp_logdir) + + self.assertAllClose(true_w, linear_model.weights, rtol=1e-2) + self.assertAllClose(true_b, linear_model.biases, rtol=1e-2) + self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) + + +class EagerLinearRegressionBenchmark(test.Benchmark): + + def benchmarkEagerLinearRegression(self): + _, _, _, _, data_gen = _create_data_gen_for_test() + + learning_rate = 0.1 + num_burnin_iters = 10 + num_iters = 200 + + device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" + logging.info("device = %s", device) + with context.device(device): + linear_model = linear_regression.LinearModel() + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + + # Perform burn-in. + linear_model.fit(data_gen.next_batch, optimizer, num_burnin_iters) + + start_time = time.time() + linear_model.fit(data_gen.next_batch, optimizer, num_iters) + wall_time = time.time() - start_time + + self.report_benchmark( + name="EagerLinearRegression", + iters=num_iters, + wall_time=wall_time) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/eager/python/examples/tests/spinn_test.py b/tensorflow/contrib/eager/python/examples/tests/spinn_test.py new file mode 100644 index 0000000000..9c8b691b98 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/tests/spinn_test.py @@ -0,0 +1,311 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import gc +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.contrib.eager.python.examples import spinn +from tensorflow.python.eager import test +from tensorflow.python.framework import test_util + + +def _generate_synthetic_snli_data_batch(sequence_length, + batch_size, + vocab_size): + """Generate a fake batch of SNLI data for testing.""" + with tf.device("cpu:0"): + labels = tf.random_uniform([batch_size], minval=1, maxval=4, dtype=tf.int64) + prem = tf.random_uniform( + (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64) + prem_trans = tf.constant(np.array( + [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3, + 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, + 3, 2, 2]] * batch_size, dtype=np.int64).T) + hypo = tf.random_uniform( + (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64) + hypo_trans = tf.constant(np.array( + [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3, + 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, + 3, 2, 2]] * batch_size, dtype=np.int64).T) + if tfe.num_gpus(): + labels = labels.gpu() + prem = prem.gpu() + prem_trans = prem_trans.gpu() + hypo = hypo.gpu() + hypo_trans = hypo_trans.gpu() + return labels, prem, prem_trans, hypo, hypo_trans + + +def _snli_classifier_config(d_embed, d_out): + config_tuple = collections.namedtuple( + "Config", ["d_hidden", "d_proj", "d_tracker", "predict", + "embed_dropout", "mlp_dropout", "n_mlp_layers", "d_mlp", + "d_out", "projection", "lr"]) + config = config_tuple( + d_hidden=d_embed, + d_proj=d_embed * 2, + d_tracker=8, + predict=False, + embed_dropout=0.1, + mlp_dropout=0.1, + n_mlp_layers=2, + d_mlp=32, + d_out=d_out, + projection=True, + lr=2e-3) + return config + + +class SpinnTest(test_util.TensorFlowTestCase): + + def setUp(self): + super(SpinnTest, self).setUp() + self._test_device = "gpu:0" if tfe.num_gpus() else "cpu:0" + + def testBundle(self): + with tf.device(self._test_device): + lstm_iter = [np.array([[0, 1], [2, 3]], dtype=np.float32), + np.array([[0, -1], [-2, -3]], dtype=np.float32), + np.array([[0, 2], [4, 6]], dtype=np.float32), + np.array([[0, -2], [-4, -6]], dtype=np.float32)] + out = spinn._bundle(lstm_iter) + + self.assertEqual(2, len(out)) + self.assertEqual(tf.float32, out[0].dtype) + self.assertEqual(tf.float32, out[1].dtype) + self.assertAllEqual(np.array([[0, 2, 0, -2, 0, 4, 0, -4]]).T, + out[0].numpy()) + self.assertAllEqual(np.array([[1, 3, -1, -3, 2, 6, -2, -6]]).T, + out[1].numpy()) + + def testUnbunbdle(self): + with tf.device(self._test_device): + state = [np.array([[0, 1, 2], [3, 4, 5]], dtype=np.float32), + np.array([[0, -1, -2], [-3, -4, -5]], dtype=np.float32)] + out = spinn._unbundle(state) + + self.assertEqual(2, len(out)) + self.assertEqual(tf.float32, out[0].dtype) + self.assertEqual(tf.float32, out[1].dtype) + self.assertAllEqual(np.array([[0, 1, 2, 0, -1, -2]]), + out[0].numpy()) + self.assertAllEqual(np.array([[3, 4, 5, -3, -4, -5]]), + out[1].numpy()) + + def testReduce(self): + with tf.device(self._test_device): + batch_size = 3 + size = 10 + tracker_size = 8 + reducer = spinn.Reduce(size, tracker_size=tracker_size) + + left_in = [] + right_in = [] + tracking = [] + for _ in range(batch_size): + left_in.append(tf.random_normal((1, size * 2))) + right_in.append(tf.random_normal((1, size * 2))) + tracking.append(tf.random_normal((1, tracker_size * 2))) + + out = reducer(left_in, right_in, tracking=tracking) + self.assertEqual(batch_size, len(out)) + self.assertEqual(tf.float32, out[0].dtype) + self.assertEqual((1, size * 2), out[0].shape) + + def testReduceTreeLSTM(self): + with tf.device(self._test_device): + size = 10 + tracker_size = 8 + reducer = spinn.Reduce(size, tracker_size=tracker_size) + + lstm_in = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]], + dtype=np.float32) + c1 = np.array([[0, 1], [2, 3]], dtype=np.float32) + c2 = np.array([[0, -1], [-2, -3]], dtype=np.float32) + + h, c = reducer._tree_lstm(c1, c2, lstm_in) + self.assertEqual(tf.float32, h.dtype) + self.assertEqual(tf.float32, c.dtype) + self.assertEqual((2, 2), h.shape) + self.assertEqual((2, 2), c.shape) + + def testTracker(self): + with tf.device(self._test_device): + batch_size = 2 + size = 10 + tracker_size = 8 + buffer_length = 18 + stack_size = 3 + + tracker = spinn.Tracker(tracker_size, False) + tracker.reset_state() + + # Create dummy inputs for testing. + bufs = [] + buf = [] + for _ in range(buffer_length): + buf.append(tf.random_normal((batch_size, size * 2))) + bufs.append(buf) + self.assertEqual(1, len(bufs)) + self.assertEqual(buffer_length, len(bufs[0])) + self.assertEqual((batch_size, size * 2), bufs[0][0].shape) + + stacks = [] + stack = [] + for _ in range(stack_size): + stack.append(tf.random_normal((batch_size, size * 2))) + stacks.append(stack) + self.assertEqual(1, len(stacks)) + self.assertEqual(3, len(stacks[0])) + self.assertEqual((batch_size, size * 2), stacks[0][0].shape) + + for _ in range(2): + out1, out2 = tracker(bufs, stacks) + self.assertIsNone(out2) + self.assertEqual(batch_size, len(out1)) + self.assertEqual(tf.float32, out1[0].dtype) + self.assertEqual((1, tracker_size * 2), out1[0].shape) + + self.assertEqual(tf.float32, tracker.state.c.dtype) + self.assertEqual((batch_size, tracker_size), tracker.state.c.shape) + self.assertEqual(tf.float32, tracker.state.h.dtype) + self.assertEqual((batch_size, tracker_size), tracker.state.h.shape) + + def testSPINN(self): + with tf.device(self._test_device): + embedding_dims = 10 + d_tracker = 8 + sequence_length = 15 + num_transitions = 27 + + config_tuple = collections.namedtuple( + "Config", ["d_hidden", "d_proj", "d_tracker", "predict"]) + config = config_tuple( + embedding_dims, embedding_dims * 2, d_tracker, False) + s = spinn.SPINN(config) + + # Create some fake data. + buffers = tf.random_normal((sequence_length, 1, config.d_proj)) + transitions = np.array( + [[3], [3], [2], [3], [3], [3], [2], [2], [2], [3], [3], [3], + [2], [3], [3], [2], [2], [3], [3], [3], [2], [2], [2], [2], + [3], [2], [2]], dtype=np.int32) + self.assertEqual(tf.int32, transitions.dtype) + self.assertEqual((num_transitions, 1), transitions.shape) + + out = s(buffers, transitions, training=True) + self.assertEqual(tf.float32, out.dtype) + self.assertEqual((1, embedding_dims), out.shape) + + def testSNLIClassifierAndTrainer(self): + with tf.device(self._test_device): + vocab_size = 40 + batch_size = 2 + d_embed = 10 + sequence_length = 15 + d_out = 4 + + config = _snli_classifier_config(d_embed, d_out) + + # Create fake embedding matrix. + embed = tf.random_normal((vocab_size, d_embed)) + + model = spinn.SNLIClassifier(config, embed) + trainer = spinn.SNLIClassifierTrainer(model, config.lr) + + (labels, prem, prem_trans, hypo, + hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length, + batch_size, + vocab_size) + + # Invoke model under non-training mode. + logits = model(prem, prem_trans, hypo, hypo_trans, training=False) + self.assertEqual(tf.float32, logits.dtype) + self.assertEqual((batch_size, d_out), logits.shape) + + # Invoke model under training model. + logits = model(prem, prem_trans, hypo, hypo_trans, training=True) + self.assertEqual(tf.float32, logits.dtype) + self.assertEqual((batch_size, d_out), logits.shape) + + # Calculate loss. + loss1 = trainer.loss(labels, logits) + self.assertEqual(tf.float32, loss1.dtype) + self.assertEqual((), loss1.shape) + + loss2, logits = trainer.train_batch( + labels, prem, prem_trans, hypo, hypo_trans) + self.assertEqual(tf.float32, loss2.dtype) + self.assertEqual((), loss2.shape) + self.assertEqual(tf.float32, logits.dtype) + self.assertEqual((batch_size, d_out), logits.shape) + # Training on the batch should have led to a change in the loss value. + self.assertNotEqual(loss1.numpy(), loss2.numpy()) + + +class EagerSpinnSNLIClassifierBenchmark(test.Benchmark): + + def benchmarkEagerSpinnSNLIClassifier(self): + test_device = "gpu:0" if tfe.num_gpus() else "cpu:0" + with tf.device(test_device): + burn_in_iterations = 2 + benchmark_iterations = 10 + + vocab_size = 1000 + batch_size = 128 + sequence_length = 15 + d_embed = 200 + d_out = 4 + + embed = tf.random_normal((vocab_size, d_embed)) + + config = _snli_classifier_config(d_embed, d_out) + model = spinn.SNLIClassifier(config, embed) + trainer = spinn.SNLIClassifierTrainer(model, config.lr) + + (labels, prem, prem_trans, hypo, + hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length, + batch_size, + vocab_size) + + for _ in range(burn_in_iterations): + trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans) + + gc.collect() + start_time = time.time() + for _ in xrange(benchmark_iterations): + trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans) + wall_time = time.time() - start_time + # Named "examples"_per_sec to conform with other benchmarks. + extras = {"examples_per_sec": benchmark_iterations / wall_time} + self.report_benchmark( + name="Eager_SPINN_SNLIClassifier_Benchmark", + iters=benchmark_iterations, + wall_time=wall_time, + extras=extras) + + +if __name__ == "__main__": + test.main() -- GitLab From 66b1f43839ccbfe7e44df004fb92d505ab6ed942 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Fri, 20 Oct 2017 15:29:41 -0700 Subject: [PATCH 222/573] Make Network compatible with eager mode. Currently it only allows to instantiate a Network in eager mode using the regular Keras API, and call it on eager tensors. PiperOrigin-RevId: 172942569 --- .../keras/_impl/keras/engine/topology.py | 2 +- .../keras/_impl/keras/integration_test.py | 4 +- tensorflow/python/keras/_impl/keras/models.py | 2 + tensorflow/python/layers/base.py | 198 +++++++++++------- tensorflow/python/layers/base_test.py | 132 +++++++++--- 5 files changed, 228 insertions(+), 110 deletions(-) diff --git a/tensorflow/python/keras/_impl/keras/engine/topology.py b/tensorflow/python/keras/_impl/keras/engine/topology.py index d9454ee8d1..c0be023b36 100644 --- a/tensorflow/python/keras/_impl/keras/engine/topology.py +++ b/tensorflow/python/keras/_impl/keras/engine/topology.py @@ -776,7 +776,7 @@ class Network(tf_base_layers.Network, Layer): if cache_key in self._output_mask_cache: return self._output_mask_cache[cache_key] else: - _, output_masks, _ = self._run_internal_graph(inputs, masks) + _, output_masks = self._run_internal_graph(inputs, masks) return output_masks def get_config(self): diff --git a/tensorflow/python/keras/_impl/keras/integration_test.py b/tensorflow/python/keras/_impl/keras/integration_test.py index d7d20e5698..7110036848 100644 --- a/tensorflow/python/keras/_impl/keras/integration_test.py +++ b/tensorflow/python/keras/_impl/keras/integration_test.py @@ -192,10 +192,12 @@ class KerasIntegrationTest(test.TestCase): model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) + self.assertEqual(len(model.losses), 2) + self.assertEqual(len(model.updates), 2) history = model.fit(x_train, y_train, epochs=10, batch_size=16, validation_data=(x_test, y_test), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.84) def test_vector_classification_shared_model(self): # Test that functional models that feature internal updates diff --git a/tensorflow/python/keras/_impl/keras/models.py b/tensorflow/python/keras/_impl/keras/models.py index 6e55c429e9..06941e4bac 100644 --- a/tensorflow/python/keras/_impl/keras/models.py +++ b/tensorflow/python/keras/_impl/keras/models.py @@ -420,6 +420,8 @@ class Sequential(Model): # Used by Layer base class. self._dtype = None self._activity_regularizer = None + self._per_input_losses = {} + self._per_input_updates = {} # The following properties are not actually used by Keras; # they exist for compatibility with TF's variable scoping mechanism. diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index 99a30657ef..91e18b2ba5 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -508,6 +508,7 @@ class Layer(object): input_list = nest.flatten(inputs) in_graph_mode = context.in_graph_mode() + in_deferred_mode = isinstance(input_list[0], _DeferredTensor) # Ensure the Layer, if being reused, is working with inputs from # the same graph as where it was created. if in_graph_mode: @@ -515,6 +516,7 @@ class Layer(object): ops._get_graph_from_inputs(input_list, graph=self.graph) # pylint: disable=protected-access except ValueError as e: raise ValueError('Input graph and Layer graph are not the same: %s' % e) + if in_graph_mode or in_deferred_mode: user_kwargs = copy.copy(kwargs) # Handle Keras mask propagation from previous layer to current layer. @@ -553,6 +555,7 @@ class Layer(object): raise ValueError('activity_regularizer currently unsupported in ' 'Eager mode. Found an activity_regularizer in ' '%s(%s).' % (self.__class__.__name__, self)) + if not in_graph_mode and not in_deferred_mode: # TODO(agarwal): support _keras_history in Eager mode. for x in input_list: if hasattr(x, '_keras_history'): @@ -581,13 +584,26 @@ class Layer(object): if call_has_scope_arg: kwargs['scope'] = scope # Check input assumptions set after layer building, e.g. input shape. - if in_graph_mode: + if in_graph_mode or in_deferred_mode: self._assert_input_compatibility(inputs) - outputs = self.call(inputs, *args, **kwargs) - if outputs is None: - raise ValueError('A layer\'s `call` method should return a Tensor ' - 'or a list of Tensors, not None.') + if not in_deferred_mode: + outputs = self.call(inputs, *args, **kwargs) + if outputs is None: + raise ValueError('A layer\'s `call` method should return a Tensor ' + 'or a list of Tensors, not None.') + else: + # Deferred mode behavior: use `_compute_output_shape` to + # infer the number of outputs of the layer and their shapes. + output_shapes = self._compute_output_shape(input_shapes) + output_shapes = nest.flatten(output_shapes) + outputs = [ + # TODO(fchollet): name the deferred tensors? + _DeferredTensor(shape=shape, dtype=self._dtype) + for shape in output_shapes + ] + if len(outputs) == 1: + outputs = outputs[0] if in_graph_mode: # Apply activity regularization. @@ -600,16 +616,18 @@ class Layer(object): activity_regularization = self._activity_regularizer(output) self.add_loss(activity_regularization) - # Handle mask computation and propagation to the next layer. - if hasattr(self, 'compute_mask'): - output_mask = self.compute_mask(inputs, previous_mask) - if isinstance(outputs, list): - if output_mask is None: - output_mask = [None for _ in range(len(outputs))] - for x, m in zip(outputs, output_mask): - x._keras_mask = m # pylint: disable=protected-access - else: - outputs._keras_mask = output_mask # pylint: disable=protected-access + if not in_deferred_mode: + # TODO(fchollet): consider how masking will work with deferred mode. + # Handle mask computation and propagation to the next layer. + if hasattr(self, 'compute_mask'): + output_mask = self.compute_mask(inputs, previous_mask) + if isinstance(outputs, list): + if output_mask is None: + output_mask = [None for _ in range(len(outputs))] + for x, m in zip(outputs, output_mask): + x._keras_mask = m # pylint: disable=protected-access + else: + outputs._keras_mask = output_mask # pylint: disable=protected-access if in_graph_mode: # If all input tensors have history metadata, @@ -631,14 +649,16 @@ class Layer(object): else: outputs = output_ls_copy + # Update global default collections. + _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) + + if in_deferred_mode or in_graph_mode: + if _have_all_keras_metadata(inputs): # Add an inbound node to the layer, so it can keep track of this call. # This updates the layer history of the output tensor(s). self._add_inbound_node( input_tensors=inputs, output_tensors=outputs, arguments=user_kwargs) - # Update global default collections. - _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) - self.built = True return outputs @@ -692,7 +712,6 @@ class Layer(object): arguments: dictionary of keyword arguments that were passed to the `call` method of the layer at the call that created the node. """ - assert context.in_graph_mode() input_tensors = nest.flatten(input_tensors) output_tensors = nest.flatten(output_tensors) @@ -1251,6 +1270,34 @@ class Node(object): } +class _DeferredTensor(object): + """Tensor-like object used to build graphs of layers in Eager mode. + + When calling a layer on a DeferredTensor, the layer will not perform any + computation and will simply perfom shape inference to return new + DeferredTensors with appropriate shape information. Thus DeferredTensor + behaves like a graph-mode Tensor when manipulated by layers. + """ + + def __init__(self, shape, dtype, name=None): + self.shape = tensor_shape.TensorShape(shape) + self.dtype = dtypes.as_dtype(dtype) + self.name = name + + def get_shape(self): + return self.shape + + def __str__(self): + return "DeferredTensor('%s', shape=%s, dtype=%s)" % (self.name, + self.get_shape(), + self.dtype.name) + + def __repr__(self): + return "<_DeferredTensor '%s' shape=%s dtype=%s>" % (self.name, + self.get_shape(), + self.dtype.name) + + class InputLayer(Layer): """Layer to be used as an entry point into a Network (a graph of layers). @@ -1283,8 +1330,6 @@ class InputLayer(Layer): input_tensor=None, sparse=False, name=None): - if context.in_eager_mode(): - raise RuntimeError('InputLayer not supported in Eager mode.') super(InputLayer, self).__init__(dtype=dtype, name=name) self.built = True self.sparse = sparse @@ -1299,16 +1344,24 @@ class InputLayer(Layer): else: batch_input_shape = None - if sparse: - input_tensor = array_ops.sparse_placeholder( + if context.in_eager_mode(): + # In eager mode, create a temporary placeholder to call the layer on. + input_tensor = _DeferredTensor( shape=batch_input_shape, dtype=dtype, name=self.name) else: - input_tensor = array_ops.placeholder( - shape=batch_input_shape, - dtype=dtype, - name=self.name) + # In graph mode, create a graph placeholder to call the layer on. + if sparse: + input_tensor = array_ops.sparse_placeholder( + shape=batch_input_shape, + dtype=dtype, + name=self.name) + else: + input_tensor = array_ops.placeholder( + shape=batch_input_shape, + dtype=dtype, + name=self.name) # For compatibility with Keras API. self.is_placeholder = True @@ -1375,8 +1428,6 @@ def Input( # pylint: disable=invalid-name Raises: RuntimeError: If called in Eager mode. """ - if context.in_eager_mode(): - raise RuntimeError('Input not supported in Eager mode.') input_layer = InputLayer( input_shape=shape, batch_size=batch_size, @@ -1440,9 +1491,10 @@ class Network(Layer): """ def __init__(self, inputs, outputs, name=None): # pylint: disable=super-init-not-called - # TODO(agarwal): Make Network work in Eager mode. if context.in_eager_mode(): - raise RuntimeError('Network not supported in Eager mode.') + # TODO(fchollet): check that all inputs and outputs are DeferredTensors. + pass + # Set layer name and scope if isinstance(name, vs.VariableScope): base_name = name.name @@ -1919,16 +1971,17 @@ class Network(Layer): masks = [None for _ in range(len(inputs))] else: masks = nest.flatten(mask) - # Try to retrieve cached outputs if the layer has already been called - # on these exact inputs. - cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) - if cache_key in self._output_tensor_cache: - # Cache hit. - return self._output_tensor_cache[cache_key] - else: - # Cache miss: actually apply the network graph to the new inputs. - output_tensors, _, _ = self._run_internal_graph(inputs, masks) - return output_tensors + + if context.in_graph_mode(): + # Try to retrieve cached outputs if the layer has already been called + # on these exact inputs. + cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) + if cache_key in self._output_tensor_cache: + # Cache hit. + return self._output_tensor_cache[cache_key] + # Actually apply the network graph to the new inputs. + outputs, _ = self._run_internal_graph(inputs, masks) + return outputs def _compute_output_shape(self, input_shape): if isinstance(input_shape, list): @@ -2091,6 +2144,7 @@ class Network(Layer): if 'mask' in estimator_util.fn_args(layer.call): if 'mask' not in kwargs: kwargs['mask'] = computed_mask + output_tensors = nest.flatten( layer.call(computed_tensor, **kwargs)) if hasattr(layer, 'compute_mask'): @@ -2121,18 +2175,19 @@ class Network(Layer): ] layer.add_loss(regularization_losses, computed_tensors) - # Update model updates and losses: - # Keep track of updates that depend on the inputs - # (e.g. BN updates). - self.add_update(layer.get_updates_for(computed_tensors), inputs) - # Keep track of unconditional updates (e.g. a counter). - self.add_update(layer.get_updates_for(None), None) - # Keep track of losses that depend on the inputs - # (e.g. activity regularizers). - self.add_loss(layer.get_losses_for(computed_tensors), inputs) - # Keep track of unconditional losses - # (e.g. weight regularizers). - self.add_loss(layer.get_losses_for(None), None) + if context.in_graph_mode(): + # Update model updates and losses: + # Keep track of updates that depend on the inputs + # (e.g. BN updates). + self.add_update(layer.get_updates_for(computed_tensors), inputs) + # Keep track of unconditional updates (e.g. a counter). + self.add_update(layer.get_updates_for(None), None) + # Keep track of losses that depend on the inputs + # (e.g. activity regularizers). + self.add_loss(layer.get_losses_for(computed_tensors), inputs) + # Keep track of unconditional losses + # (e.g. weight regularizers). + self.add_loss(layer.get_losses_for(None), None) # Update tensor_map. for x, y, mask in zip(reference_output_tensors, output_tensors, @@ -2149,31 +2204,26 @@ class Network(Layer): output_tensors.append(tensor) output_masks.append(mask) - # Update cache; - # keys are based on ids on input tensors and inputs masks. - cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) - if len(output_tensors) == 1: output_tensors = output_tensors[0] - self._output_tensor_cache[cache_key] = output_tensors - else: - self._output_tensor_cache[cache_key] = output_tensors - - if len(output_masks) == 1: - output_masks = output_masks[0] - self._output_mask_cache[cache_key] = output_masks - else: - self._output_mask_cache[cache_key] = output_masks - - if output_shapes is not None: - input_shapes = [_static_shape(x) for x in inputs] - cache_key = _object_list_uid(input_shapes) - if len(output_shapes) == 1: + if output_shapes is not None: output_shapes = output_shapes[0] + if output_masks is not None: + output_masks = output_masks[0] + + if context.in_graph_mode(): + # Update cache; + # keys are based on ids on input tensors and inputs masks. + cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) + self._output_tensor_cache[cache_key] = output_tensors + if output_masks is not None: + self._output_mask_cache[cache_key] = output_masks + if output_shapes is not None: + input_shapes = [_static_shape(x) for x in inputs] + cache_key = _object_list_uid(input_shapes) self._output_shape_cache[cache_key] = output_shapes - else: - self._output_shape_cache[cache_key] = output_shapes - return output_tensors, output_masks, output_shapes + + return output_tensors, output_masks def _is_tensor_or_tensor_list(v): diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index 813a2fe755..71eff2f965 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import copy +import numpy as np + from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -41,13 +43,13 @@ class BaseLayerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes() def testLayerProperties(self): layer = base_layers.Layer(name='my_layer') - self.assertListEqual(layer.variables, []) - self.assertListEqual(layer.trainable_variables, []) - self.assertListEqual(layer.non_trainable_variables, []) + self.assertEqual(layer.variables, []) + self.assertEqual(layer.trainable_variables, []) + self.assertEqual(layer.non_trainable_variables, []) if context.in_graph_mode(): # updates, losses only suppported in GRAPH mode - self.assertListEqual(layer.updates, []) - self.assertListEqual(layer.losses, []) + self.assertEqual(layer.updates, []) + self.assertEqual(layer.losses, []) self.assertEqual(layer.built, False) layer = base_layers.Layer(name='my_layer', trainable=False) self.assertEqual(layer.trainable, False) @@ -60,11 +62,11 @@ class BaseLayerTest(test.TestCase): variable = layer.add_variable( 'my_var', [2, 2], initializer=init_ops.zeros_initializer()) self.assertEqual(variable.name, 'my_layer/my_var:0') - self.assertListEqual(layer.variables, [variable]) - self.assertListEqual(layer.trainable_variables, [variable]) - self.assertListEqual(layer.non_trainable_variables, []) + self.assertEqual(layer.variables, [variable]) + self.assertEqual(layer.trainable_variables, [variable]) + self.assertEqual(layer.non_trainable_variables, []) if context.in_graph_mode(): - self.assertListEqual( + self.assertEqual( layer.variables, ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) @@ -74,9 +76,9 @@ class BaseLayerTest(test.TestCase): 'non_trainable_var', [2, 2], initializer=init_ops.zeros_initializer(), trainable=False) - self.assertListEqual(layer.variables, [variable, variable_2]) - self.assertListEqual(layer.trainable_variables, [variable]) - self.assertListEqual(layer.non_trainable_variables, [variable_2]) + self.assertEqual(layer.variables, [variable, variable_2]) + self.assertEqual(layer.trainable_variables, [variable]) + self.assertEqual(layer.non_trainable_variables, [variable_2]) if context.in_graph_mode(): self.assertEqual( len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 1) @@ -105,8 +107,8 @@ class BaseLayerTest(test.TestCase): inputs = random_ops.random_uniform((5,), seed=1) layer.apply(inputs) layer.apply(inputs) - self.assertListEqual([v.name for v in layer.variables], - ['my_layer/my_var:0']) + self.assertEqual([v.name for v in layer.variables], + ['my_layer/my_var:0']) # Creating a layer with no scope leads to lazy construction of # the scope at apply() time. It uses scope "/base_name" @@ -120,7 +122,7 @@ class BaseLayerTest(test.TestCase): # The variables were created outside of the Layer, and # reuse=True, so the Layer does not own them and they are not # stored in its collection. - self.assertListEqual(lazy_layer.variables, []) + self.assertEqual(lazy_layer.variables, []) self.assertEqual(lazy_layer._scope.name, 'new_scope/my_layer') # Creating a layer with no scope leads to lazy construction of @@ -135,7 +137,7 @@ class BaseLayerTest(test.TestCase): # The variables were created outside of the Layer, and # reuse=True, so the Layer does not own them and they are not # stored in its collection. - self.assertListEqual(lazy_layer.variables, []) + self.assertEqual(lazy_layer.variables, []) self.assertEqual(lazy_layer._scope.name, 'new_scope') # Checking for graph equality is only done in GRAPH mode. @@ -183,14 +185,14 @@ class BaseLayerTest(test.TestCase): outputs = layer.apply(inputs) self.assertEqual(layer.built, True) self.assertEqual(outputs.op.name, 'my_layer/add') - self.assertListEqual([v.name - for v in layer.variables], ['my_layer/my_var:0']) + self.assertEqual([v.name + for v in layer.variables], ['my_layer/my_var:0']) with self.assertRaisesRegexp(ValueError, 'my_layer/this_will_break_on_second_call'): layer.apply(inputs) # The list of variables hasn't changed. - self.assertListEqual([v.name - for v in layer.variables], ['my_layer/my_var:0']) + self.assertEqual([v.name + for v in layer.variables], ['my_layer/my_var:0']) @test_util.run_in_graph_and_eager_modes() def testDeepCopy(self): @@ -435,8 +437,8 @@ class BaseLayerTest(test.TestCase): dense_layer.add_update(0, inputs=a) dense_layer.add_update(1, inputs=None) - self.assertListEqual(dense_layer.get_updates_for(a), [0]) - self.assertListEqual(dense_layer.get_updates_for(None), [1]) + self.assertEqual(dense_layer.get_updates_for(a), [0]) + self.assertEqual(dense_layer.get_updates_for(None), [1]) def test_get_losses_for(self): a = base_layers.Input(shape=(2,)) @@ -444,8 +446,8 @@ class BaseLayerTest(test.TestCase): dense_layer.add_loss(0, inputs=a) dense_layer.add_loss(1, inputs=None) - self.assertListEqual(dense_layer.get_losses_for(a), [0]) - self.assertListEqual(dense_layer.get_losses_for(None), [1]) + self.assertEqual(dense_layer.get_losses_for(a), [0]) + self.assertEqual(dense_layer.get_losses_for(None), [1]) def testTopologicalAttributes(self): # test layer attributes / methods related to cross-layer connectivity. @@ -612,7 +614,7 @@ class NetworkTest(test.TestCase): a = base_layers.Input(shape=(32,), name='input_a') b = base_layers.Input(shape=(32,), name='input_b') - self.assertListEqual(a.get_shape().as_list(), [None, 32]) + self.assertEqual(a.get_shape().as_list(), [None, 32]) a_layer, a_node_index, a_tensor_index = a._keras_history b_layer, _, _ = b._keras_history self.assertEqual(len(a_layer._inbound_nodes), 1) @@ -620,11 +622,11 @@ class NetworkTest(test.TestCase): node = a_layer._inbound_nodes[a_node_index] self.assertEqual(node.outbound_layer, a_layer) - self.assertListEqual(node.inbound_layers, []) - self.assertListEqual(node.input_tensors, [a]) - self.assertListEqual(node.input_shapes, [(None, 32)]) - self.assertListEqual(node.output_tensors, [a]) - self.assertListEqual(node.output_shapes, [(None, 32)]) + self.assertEqual(node.inbound_layers, []) + self.assertEqual(node.input_tensors, [a]) + self.assertEqual(node.input_shapes, [(None, 32)]) + self.assertEqual(node.output_tensors, [a]) + self.assertEqual(node.output_shapes, [(None, 32)]) dense = core_layers.Dense(16, name='dense_1') dense(a) @@ -632,12 +634,12 @@ class NetworkTest(test.TestCase): self.assertEqual(len(dense._inbound_nodes), 2) self.assertEqual(len(dense._outbound_nodes), 0) - self.assertListEqual(dense._inbound_nodes[0].inbound_layers, [a_layer]) + self.assertEqual(dense._inbound_nodes[0].inbound_layers, [a_layer]) self.assertEqual(dense._inbound_nodes[0].outbound_layer, dense) - self.assertListEqual(dense._inbound_nodes[1].inbound_layers, [b_layer]) + self.assertEqual(dense._inbound_nodes[1].inbound_layers, [b_layer]) self.assertEqual(dense._inbound_nodes[1].outbound_layer, dense) - self.assertListEqual(dense._inbound_nodes[0].input_tensors, [a]) - self.assertListEqual(dense._inbound_nodes[1].input_tensors, [b]) + self.assertEqual(dense._inbound_nodes[0].input_tensors, [a]) + self.assertEqual(dense._inbound_nodes[1].input_tensors, [b]) # Test config config_0 = dense._inbound_nodes[0].get_config() @@ -889,5 +891,67 @@ class NetworkTest(test.TestCase): self.assertAllEqual(self.evaluate(a * mask), self.evaluate(b)) +class DeferredModeTest(test.TestCase): + + def testDeferredTensorAttributes(self): + x = base_layers._DeferredTensor(shape=(None, 2), dtype='float32', name='x') + self.assertEqual(str(x), + 'DeferredTensor(\'x\', shape=(?, 2), dtype=float32)') + self.assertEqual(repr(x), + '<_DeferredTensor \'x\' shape=(?, 2) dtype=float32>') + + @test_util.run_in_graph_and_eager_modes() + def testSimpleNetworkBuilding(self): + inputs = base_layers.Input(shape=(32,)) + if context.in_eager_mode(): + self.assertIsInstance(inputs, base_layers._DeferredTensor) + self.assertEqual(inputs.dtype.name, 'float32') + self.assertEqual(inputs.shape.as_list(), [None, 32]) + + x = core_layers.Dense(2)(inputs) + if context.in_eager_mode(): + self.assertIsInstance(x, base_layers._DeferredTensor) + self.assertEqual(x.dtype.name, 'float32') + self.assertEqual(x.shape.as_list(), [None, 2]) + + outputs = core_layers.Dense(4)(x) + network = base_layers.Network(inputs, outputs) + self.assertIsInstance(network, base_layers.Network) + + if context.in_eager_mode(): + # It should be possible to call such a network on EagerTensors. + inputs = constant_op.constant( + np.random.random((10, 32)).astype('float32')) + outputs = network(inputs) + self.assertEqual(outputs.shape.as_list(), [10, 4]) + + @test_util.run_in_graph_and_eager_modes() + def testMultiIONetworkbuilding(self): + input_a = base_layers.Input(shape=(32,)) + input_b = base_layers.Input(shape=(16,)) + a = core_layers.Dense(16)(input_a) + + class AddLayer(base_layers.Layer): + + def call(self, inputs): + return inputs[0] + inputs[1] + + def _compute_output_shape(self, input_shape): + return input_shape[0] + + c = AddLayer()([a, input_b]) # pylint: disable=not-callable + c = core_layers.Dense(2)(c) + + network = base_layers.Network([input_a, input_b], [a, c]) + if context.in_eager_mode(): + a_val = constant_op.constant( + np.random.random((10, 32)).astype('float32')) + b_val = constant_op.constant( + np.random.random((10, 16)).astype('float32')) + outputs = network([a_val, b_val]) + self.assertEqual(len(outputs), 2) + self.assertEqual(outputs[0].shape.as_list(), [10, 16]) + self.assertEqual(outputs[1].shape.as_list(), [10, 2]) + if __name__ == '__main__': test.main() -- GitLab From 703182d854e704ba32770342a2cac28022f7814d Mon Sep 17 00:00:00 2001 From: Mingxing Tan Date: Fri, 20 Oct 2017 15:33:13 -0700 Subject: [PATCH 223/573] Add performance guide for fused decode_and_crop_jpeg optimization. PiperOrigin-RevId: 172943116 --- .../docs_src/performance/performance_guide.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md index 30fb91f9d9..06bb40f64d 100644 --- a/tensorflow/docs_src/performance/performance_guide.md +++ b/tensorflow/docs_src/performance/performance_guide.md @@ -87,6 +87,40 @@ the Dataset API is still strongly recommended. Try to avoid the following: sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) ``` +#### Fused decode and crop + +If inputs are JPEG images that also require cropping, use fused +@{tf.image.decode_and_crop_jpeg} to speed up preprocessing. +`tf.image.decode_and_crop_jpeg` only decodes the part of +the image within the crop window. This significantly speeds up the process if +the crop window is much smaller than the full image. For imagenet data, this +approach could speed up the input pipeline by up to 30%. + +Example Usage: + +```python +def _image_preprocess_fn(image_buffer): + # image_buffer 1-D string Tensor representing the raw JPEG image buffer. + + # Extract image shape from raw JPEG image buffer. + image_shape = tf.image.extract_jpeg_shape(image_buffer) + + # Get a crop window with distorted bounding box. + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + image_shape, ...) + bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box + + # Decode and crop image. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + cropped_image = tf.image.decode_and_crop_jpeg(image, crop_window) +``` + +`tf.image.decode_and_crop_jpeg` is available on all platforms. There is no speed +up on Windows due to the use of `libjpeg` vs. `libjpeg-turbo` on other +platforms. + #### Use large files Reading large numbers of small files significantly impacts I/O performance. -- GitLab From 985031a10194c219f7e5f532a703b8b07e85faac Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 20 Oct 2017 15:35:10 -0700 Subject: [PATCH 224/573] Allows tfe.enable_eager_execution(device_policy=tfe.DEVICE_POLICY_WARN). PiperOrigin-RevId: 172943398 --- tensorflow/contrib/eager/README.OPENSOURCE.md | 15 - tensorflow/contrib/eager/README.md | 74 +- .../contrib/eager/python/examples/BUILD | 134 --- .../eager/python/examples/cart_pole.py | 282 ------ .../eager/python/examples/cart_pole_helper.py | 60 -- .../python/examples/linear_regression.py | 197 ---- .../python/examples/notebooks/1_basics.ipynb | 529 ----------- .../examples/notebooks/2_gradients.ipynb | 864 ------------------ .../examples/notebooks/3_datasets.ipynb | 218 ----- .../examples/tests/cart_pole_helper_test.py | 51 -- .../python/examples/tests/cart_pole_test.py | 162 ---- .../examples/tests/linear_regression_test.py | 114 --- .../eager/python/examples/tests/spinn_test.py | 311 ------- tensorflow/contrib/eager/python/tfe.py | 7 + tensorflow/python/eager/context.py | 23 +- tensorflow/python/eager/ops_test.py | 17 + tensorflow/python/framework/ops.py | 32 +- tensorflow/python/pywrap_tfe.i | 6 + 18 files changed, 92 insertions(+), 3004 deletions(-) delete mode 100644 tensorflow/contrib/eager/README.OPENSOURCE.md delete mode 100644 tensorflow/contrib/eager/python/examples/BUILD delete mode 100644 tensorflow/contrib/eager/python/examples/cart_pole.py delete mode 100644 tensorflow/contrib/eager/python/examples/cart_pole_helper.py delete mode 100644 tensorflow/contrib/eager/python/examples/linear_regression.py delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py delete mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py delete mode 100644 tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py delete mode 100644 tensorflow/contrib/eager/python/examples/tests/spinn_test.py diff --git a/tensorflow/contrib/eager/README.OPENSOURCE.md b/tensorflow/contrib/eager/README.OPENSOURCE.md deleted file mode 100644 index a4a3af08cf..0000000000 --- a/tensorflow/contrib/eager/README.OPENSOURCE.md +++ /dev/null @@ -1,15 +0,0 @@ -TensorFlow has many kernels for doing (deep) learning and data manipulation. -There are typically assembled into computational graphs which can run -efficiently in a variety of environments. - -We are exploring an alternative interaction, where kernels are invoked -immediately and call this "eager execution". We are hoping to retain the -benefits of graphs while improving usability with benefits like: - -- Immediate error messages and easier debugging -- Flexibility to use Python datastructures and control flow -- Reduced boilerplate - -Eager execution is under active development. -There are not many developer-facing materials yet, but stay tuned for updates -in this directory. diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md index fe577fa7eb..a4a3af08cf 100644 --- a/tensorflow/contrib/eager/README.md +++ b/tensorflow/contrib/eager/README.md @@ -1,65 +1,15 @@ -# TensorFlow Eager Execution +TensorFlow has many kernels for doing (deep) learning and data manipulation. +There are typically assembled into computational graphs which can run +efficiently in a variety of environments. -> *WARNING*: This is a preview/pre-alpha version. The API and performance -> characteristics are subject to change. +We are exploring an alternative interaction, where kernels are invoked +immediately and call this "eager execution". We are hoping to retain the +benefits of graphs while improving usability with benefits like: +- Immediate error messages and easier debugging +- Flexibility to use Python datastructures and control flow +- Reduced boilerplate -Eager execution is an experimental interface to TensorFlow that provides an -imperative programming style (à la [NumPy](http://www.numpy.org)). When you -enable eager execution, TensorFlow operations execute immediately; you do not -execute a pre-constructed graph with -[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session). - -For example, consider a simple computation in TensorFlow: - -```python -x = tf.placeholder(tf.float32, shape=[1, 1]) -m = tf.matmul(x, x) - -with tf.Session() as sess: - print(sess.run(m, feed_dict={x: [[2.]]})) - -# Will print [[4.]] -``` - -Eager execution makes this much simpler: - -```python -x = [[2.]] -m = tf.matmul(x, x) - -print(m) -``` - -## Installation - -Since eager execution is not yet part of a TensorFlow release, using it requires -either [building from source](https://www.tensorflow.org/install/install_sources) -or the latest nightly builds. The nightly builds are available as: - -- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and - -- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images. - -For example, to run the latest nightly docker image: - -```sh -# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker -nvidia-docker pull tensorflow/tensorflow:nightly-gpu -nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu - -# If you do not have a GPU, use the CPU-only image -docker pull tensorflow/tensorflow:nightly -docker run -it -p 8888:8888 tensorflow/tensorflow:nightly -``` - -And then visit http://localhost:8888 in your browser for a Jupyter notebook -environment. Try out the notebooks below. - -## Documentation - -For an introduction to TensorFlow eager execution, see the Jupyter notebooks: - -- [Basic Usage](examples/notebooks/1_basics.ipynb) -- [Gradients](examples/notebooks/2_gradients.ipynb) -- [Importing Data](examples/notebooks/3_datasets.ipynb) +Eager execution is under active development. +There are not many developer-facing materials yet, but stay tuned for updates +in this directory. diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD deleted file mode 100644 index 3604139819..0000000000 --- a/tensorflow/contrib/eager/python/examples/BUILD +++ /dev/null @@ -1,134 +0,0 @@ -# Description: -# Open-source examples and tutorials for TensorFlow Eager Execution. - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -load("//tensorflow:tensorflow.bzl", "py_test") -load("//tensorflow:tensorflow.bzl", "cuda_py_test") - -py_binary( - name = "linear_regression", - srcs = ["linear_regression.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "@six_archive//:six", - ], -) - -py_library( - name = "cart_pole_helper", - srcs = ["cart_pole_helper.py"], - srcs_version = "PY2AND3", -) - -py_library( - name = "spinn", - srcs = ["spinn.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -py_binary( - name = "cart_pole", - srcs = ["cart_pole.py"], - srcs_version = "PY2AND3", - deps = [ - ":cart_pole_helper", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "@six_archive//:six", - ], -) - -py_binary( - name = "spinn_prep_data", - srcs = ["spinn_prep_data.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow:tensorflow_py", - ], -) - -py_binary( - name = "spinn_train", - srcs = ["spinn_train.py"], - srcs_version = "PY2AND3", - deps = [ - ":spinn", - "//tensorflow:tensorflow_py", - "//tensorflow/contrib/eager/python:tfe", - "//third_party/py/numpy", - "@six_archive//:six", - ], -) - -cuda_py_test( - name = "linear_regression_test", - size = "small", - srcs = ["tests/linear_regression_test.py"], - additional_deps = [ - ":linear_regression", - "//tensorflow/python/eager:test", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - ], -) - -py_test( - name = "cart_pole_helper_test", - srcs = ["tests/cart_pole_helper_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":cart_pole_helper", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - ], -) - -cuda_py_test( - name = "cart_pole_test", - size = "small", - srcs = ["tests/cart_pole_test.py"], - additional_deps = [ - ":cart_pole", - "//tensorflow/python/eager:test", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:training", - ], -) - -cuda_py_test( - name = "spinn_test", - size = "medium", - srcs = ["tests/spinn_test.py"], - additional_deps = [ - ":spinn", - "//third_party/py/numpy", - "//tensorflow:tensorflow_py", - "//tensorflow/python/eager:test", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - ], -) - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), - visibility = ["//tensorflow:__subpackages__"], -) diff --git a/tensorflow/contrib/eager/python/examples/cart_pole.py b/tensorflow/contrib/eager/python/examples/cart_pole.py deleted file mode 100644 index 56235e4039..0000000000 --- a/tensorflow/contrib/eager/python/examples/cart_pole.py +++ /dev/null @@ -1,282 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -r"""TensorFlow Eager Execution Example: OpenAI Gym CartPole. - -Solves the cart-pole problem with policy gradient-based reinforcement learning. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -import gym -import numpy as np -from six.moves import input # pylint: disable=redefined-builtin -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -from tensorflow.contrib.eager.python import tfe -from tensorflow.contrib.eager.python.examples import cart_pole_helper - - -class PolicyNetwork(object): - """Policy network for the cart-pole reinforcement learning problem. - - The forward path of the network takes an observation from the cart-pole - environment (length-4 vector) and outputs an action. - """ - - def __init__(self, hidden_size, train_logdir=None): - """Constructor of PolicyNetwork. - - Args: - hidden_size: Size of the hidden layer, as an `int`. - train_logdir: The directory in which summaries will be written for - TensorBoard during training (optional). - """ - self._hidden_layer = tf.layers.Dense(hidden_size, activation=tf.nn.elu) - self._output_layer = tf.layers.Dense(1) - - # Gradient function. - self._grad_fn = tfe.implicit_gradients( - self._get_cross_entropy_and_save_actions) - - # Support for TensorBoard summaries. Once training has started, use: - # tensorboard --logdir= - self._summary_writer = (tfe.SummaryWriter(train_logdir) if train_logdir - else None) - - def forward(self, inputs): - """Given inputs, calculate logits and action. - - Args: - inputs: Observations from a step in the cart-pole environment, of shape - `(batch_size, input_size)` - - Returns: - logits: the logits output by the output layer. This can be viewed as the - likelihood vales of choosing the left (0) action. Shape: - `(batch_size, 1)`. - actions: randomly selected actions ({0, 1}) based on the logits. Shape: - `(batch_size, 1)`. - """ - hidden = self._hidden_layer(inputs) - logits = self._output_layer(hidden) - - # Probability of selecting the left action. - left_p = tf.nn.sigmoid(logits) - # Probabilities of selecting the left and right actions. - left_right_ps = tf.concat([left_p, 1.0 - left_p], 1) - # Randomly-generated actions based on the probabilities. - actions = tf.multinomial(tf.log(left_right_ps), 1) - return logits, actions - - def _get_cross_entropy_and_save_actions(self, inputs): - """Given inputs, get the sigmoid cross entropy and save selection action. - - Args: - inputs: Observation from a step in the cart-pole environment. - - Returns: - The sigmoid cross-entropy loss given the selected action and logits, based - on the assumption that the selected action was rewarded by the - environment. - """ - logits, actions = self.forward(inputs) - - # N.B.: This is an important step. We save the value of the `actions` in a - # member variable for use with the RL environment. In classic TensorFlow - # (non-eager execution), it is less straightfoward to access intermediate - # computation results in this manner (c.f., `tf.Session.partial_run()`). - self._current_actions = actions - - labels = 1.0 - tf.cast(actions, tf.float32) - return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits) - - def train(self, - cart_pole_env, - optimizer, - discount_rate, - num_games, - max_steps_per_game): - """Train the PolicyNetwork by playing `num_games` games in `cart_pole_env`. - - Arguments: - cart_pole_env: The cart-pole gym environment object. - optimizer: A TensorFlow `Optimizer` object to be used in this training - (e.g., `tf.train.AdamOptimizer`). - discount_rate: Reward discounting rate. - num_games: Number of games to run per parameter update. - max_steps_per_game: Maximum number of steps to run in each game. - - Returns: - Step counts from all games, as a `list` of `int`. - """ - all_gradient_lists = [] - all_rewards = [] - for _ in xrange(num_games): - obs = cart_pole_env.reset() - game_rewards = [] - game_gradient_lists = [] - for _ in xrange(max_steps_per_game): - # TODO(cais): Can we save the tf.constant() call? - grad_list, var_list = zip(*self._grad_fn(tf.constant([obs]))) - game_gradient_lists.append(grad_list) - - action = self._current_actions.numpy()[0][0] - obs, reward, done, _ = cart_pole_env.step(action) - game_rewards.append(reward) - if reward != 1.0 or done: - break - - all_gradient_lists.append(game_gradient_lists) - all_rewards.append(game_rewards) - - normalized_rewards = cart_pole_helper.discount_and_normalize_rewards( - all_rewards, discount_rate) - all_grads_and_vars = self._scale_and_average_gradients(var_list, - all_gradient_lists, - normalized_rewards) - optimizer.apply_gradients(all_grads_and_vars) - step_counts = [len(rewards) for rewards in all_rewards] - - if self._summary_writer: - self._summary_writer.scalar("mean_step_count", np.mean(step_counts)) - self._summary_writer.step() - - return step_counts - - def _scale_and_average_gradients(self, - variable_list, - all_gradient_lists, - normalized_rewards): - """Scale gradient tensors with normalized rewards.""" - num_games = len(all_gradient_lists) - grads_and_vars = [] - for j, var in enumerate(variable_list): - scaled_gradients = [] - for g in xrange(int(num_games)): - num_steps = len(all_gradient_lists[g]) - for s in xrange(num_steps): - scaled_gradients.append( - all_gradient_lists[g][s][j] * normalized_rewards[g][s]) - mean_scaled_gradients = sum(scaled_gradients) / len(scaled_gradients) - grads_and_vars.append((mean_scaled_gradients, var)) - return grads_and_vars - - def play(self, cart_pole_env, max_steps=None, render=False): - """Play a game in the cart-pole gym environment. - - Args: - cart_pole_env: The cart-pole gym environment object. - max_steps: Maximum number of steps to run in the game. - render: Whether the game state is to be rendered on the screen. - """ - if render: - input("\nAbout to play a game with rendering. Press Enter to continue: ") - - steps = 0 - obs = cart_pole_env.reset() - while True: - # TODO(cais): Can we save the tf.constant() call? - _, actions = self.forward(tf.constant([obs])) - if render: - cart_pole_env.render() - obs, reward, done, _ = cart_pole_env.step(actions.numpy()[0][0]) - steps += 1 - if done or reward != 1.0 or max_steps is not None and steps >= max_steps: - break - - -def main(_): - tf.set_random_seed(0) - - cart_pole_env = gym.make("CartPole-v0") - cart_pole_env.seed(0) - cart_pole_env.reset() - - device = "gpu:0" if tfe.num_gpus() else "cpu:0" - print("Using device: %s" % device) - - with tf.device(device): - policy_network = PolicyNetwork(FLAGS.hidden_size, train_logdir=FLAGS.logdir) - optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate) - - # Training loop. - for i in xrange(FLAGS.num_iterations): - step_counts = policy_network.train( - cart_pole_env, - optimizer, - FLAGS.discount_rate, - FLAGS.num_games_per_iteration, - FLAGS.max_steps_per_game) - print("Iteration %d: step counts = %s; mean = %g" % ( - i, step_counts, np.mean(step_counts))) - sys.stdout.flush() - - # Optional playing after training, with rendering. - if FLAGS.play_after_training: - policy_network.play(cart_pole_env, - max_steps=FLAGS.max_steps_per_game, - render=True) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--hidden_size", - type=int, - default=5, - help="Size of the hidden layer of the policy network.") - parser.add_argument( - "--discount_rate", - type=float, - default=0.95, - help="Reward discounting rate.") - parser.add_argument( - "--learning_rate", - type=float, - default=0.05, - help="Learning rate to be used during training.") - parser.add_argument( - "--num_iterations", - type=int, - default=100, - help="Number of training iterations.") - parser.add_argument( - "--num_games_per_iteration", - type=int, - default=20, - help="Number of games to run in each training iteration.") - parser.add_argument( - "--max_steps_per_game", - type=int, - default=1000, - help="Maximum number of steps to run in each game.") - parser.add_argument( - "--logdir", - type=str, - default=None, - help="logdir in which TensorBoard summaries will be written (optional).") - parser.add_argument( - "--play_after_training", - action="store_true", - help="Play a game after training (with rendering).") - - FLAGS, unparsed = parser.parse_known_args() - tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/cart_pole_helper.py b/tensorflow/contrib/eager/python/examples/cart_pole_helper.py deleted file mode 100644 index 1b80f90165..0000000000 --- a/tensorflow/contrib/eager/python/examples/cart_pole_helper.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Helper functions for reinforcement learning in the cart-pole problem.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - - -def discount_rewards(rewards, discount_rate): - """Discout reward values with discount rate. - - Args: - rewards: A sequence of reward values in time. - discount_rate: (`float`) reward discounting rate (e.g., 0.95). - - Returns: - Discounted reward values. - """ - discounted = [] - for reward in reversed(rewards): - discounted.append( - (discounted[-1] if discounted else 0.0) * discount_rate + reward) - return list(reversed(discounted)) - - -def discount_and_normalize_rewards(reward_sequences, discount_rate): - """Perform discounting on a number of reward sequences; then normalize values. - - Args: - reward_sequences: an `iterable` of reward sequences. - discount_rate: reward discounting rate (e.g., 0.95). - - Returns: - A `list` of reward value `list`s, discounted and normalized. - """ - discounted = [] - for sequence in reward_sequences: - discounted.append(discount_rewards(sequence, discount_rate)) - discounted = np.array(discounted) - - # Compute overall mean and stddev. - flattened = np.concatenate(discounted) - mean = np.mean(flattened) - std = np.std(flattened) - return [((d - mean) / std) for d in discounted] diff --git a/tensorflow/contrib/eager/python/examples/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression.py deleted file mode 100644 index 538d6d4225..0000000000 --- a/tensorflow/contrib/eager/python/examples/linear_regression.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -# pylint: disable=line-too-long -r"""TensorFlow Eager Execution Example: Linear Regression. - -This example shows how to use TensorFlow Eager Execution to fit a simple linear -regression model using some synthesized data. Specifically, it illustrates how -to define the forward path of the linear model and the loss function, as well -as how to obtain the gradients of the loss function with respect to the -variables and update the variables with the gradients. -""" -# pylint: enable=line-too-long - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import sys - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf - -# TODO(cais): Use tf.contrib.eager namespace when ready. -from tensorflow.contrib.eager.python import tfe - - -class DataGenerator(object): - """Generates synthetic data for linear regression.""" - - def __init__(self, w, b, noise_level, batch_size): - self._w = w - self._b = b - self._noise_level = noise_level - self._batch_size = batch_size - self._ndims = w.shape[0] - - def next_batch(self): - """Generate a synthetic batch of xs and ys.""" - xs = tf.random_normal([self._batch_size, self._ndims]) - ys = (tf.matmul(xs, self._w) + self._b + - self._noise_level * tf.random_normal([self._batch_size, 1])) - return xs, ys - - -class LinearModel(object): - """A TensorFlow linear regression model. - - Uses TensorFlow's eager execution. - - For those familiar with TensorFlow graphs, notice the absence of - `tf.Session`. The `forward()` method here immediately executes and - returns output values. The `loss()` method immediately compares the - output of `forward()` with the target adn returns the MSE loss value. - The `fit()` performs gradient-descent training on the model's weights - and bias. - """ - - def __init__(self): - """Constructs a LinearModel object.""" - self._hidden_layer = tf.layers.Dense(1) - - # loss_value_and_grad_fn is a function that when invoked, will return the - # loss value and the gradients of loss with respect to the variables. It has - # the same input arguments as `self.loss()`. - self._loss_value_and_grad_fn = tfe.implicit_value_and_gradients(self.loss) - - @property - def weights(self): - """Get values of weights as a numpy array.""" - return self._hidden_layer.variables[0].read_value().numpy() - - @property - def biases(self): - """Get values of biases as a numpy array.""" - return self._hidden_layer.variables[1].read_value().numpy() - - def forward(self, xs): - """Invoke the linear model. - - Args: - xs: input features, as a tensor of size [batch_size, ndims]. - - Returns: - ys: the predictions of the linear mode, as a tensor of size [batch_size] - """ - # Note: Unlike classic TensorFlow, operations such as self._hidden_layer - # will execute the underlying computation immediately. - return self._hidden_layer(xs) - - def loss(self, xs, ys): - """Loss of the linear model. - - Args: - xs: input features, as a tensor of size [batch_size, ndims]. - ys: the target values of y, as a tensor of size [batch_size]. - - Returns: - The mean square error loss value. - """ - return tf.reduce_mean(tf.square(self.forward(xs) - ys)) - - def fit(self, - batch_fn, - optimizer, - num_iters, - verbose=False, - logdir=None): - """Fit the linear-regression model. - - Args: - batch_fn: A function, which when called without any arguments, returns a - batch of xs and ys for training. - optimizer: The TensorFlow Optimizer object to be used. - num_iters: Number of training iterations to perform. - verbose: If true, will print out loss values at every iteration. - logdir: The directory in which summaries will be written for TensorBoard - (optional). - """ - if logdir: - # Support for TensorBoard summaries. Once training has started, use: - # tensorboard --logdir= - summary_writer = tfe.SummaryWriter(logdir) - - # Training loop. - for i in xrange(num_iters): - # Generate a (mini-)batch of data for training. - xs, ys = batch_fn() - - # Call the function obtained above to get the loss and gradient values at - # the specific training batch. The function has the same input arguments - # as the forward function, i.e., `linear_loss()`. - loss_value, grads_and_vars = self._loss_value_and_grad_fn(xs, ys) - if verbose: - print("Iteration %d: loss = %s" % (i, loss_value.numpy())) - - # Send the gradients to the optimizer and update the Variables, i.e., `w` - # and `b`. - optimizer.apply_gradients(grads_and_vars) - - if logdir: - summary_writer.scalar("loss", loss_value) - summary_writer.step() - - -def main(_): - # Ground-truth constants. - true_w = np.array([[-2.0], [4.0], [1.0]], dtype=np.float32) - true_b = np.array([0.5], dtype=np.float32) - noise_level = 0.01 - - # Training constants. - batch_size = 64 - learning_rate = 0.1 - num_iters = 20 - - print("True w: %s" % true_w) - print("True b: %s\n" % true_b) - - device = "gpu:0" if tfe.num_gpus() else "cpu:0" - print("Using device: %s" % device) - with tf.device(device): - linear_model = LinearModel() - - optimizer = tf.train.GradientDescentOptimizer(learning_rate) - data_gen = DataGenerator(true_w, true_b, noise_level, batch_size) - linear_model.fit(data_gen.next_batch, optimizer, num_iters, verbose=True, - logdir=FLAGS.logdir) - - print("\nAfter training: w = %s" % linear_model.weights) - print("\nAfter training: b = %s" % linear_model.biases) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--logdir", - type=str, - default=None, - help="logdir in which TensorBoard summaries will be written (optional).") - FLAGS, unparsed = parser.parse_known_args() - - # Use tfe.run() instead of tf.app.run() for eager execution. - tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb deleted file mode 100644 index 9c2e6f15b4..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb +++ /dev/null @@ -1,529 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# Eager Execution Tutorial: Basics\n", - "\n", - "This notebook introduces the basics of using TensorFlow's eager execution capabilities. It covers concepts such as:\n", - "\n", - "* Importing required packages\n", - "* Enabling eager execution\n", - "* Creating and using TensorFlow Tensors and Variables\n", - "* Using TensorFlow interactively\n", - "* Using GPUs with eager execution enabled\n", - "\n", - "This notebook does *not* cover modeling topics, such as gradients." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "# Step 1: Import Eager\n", - "\n", - "The key imports for eager execution are the following:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RlIWhyeLoYnG" - }, - "outputs": [], - "source": [ - "# Import TensorFlow.\n", - "import tensorflow as tf\n", - "\n", - "# Import TensorFlow eager execution support (subject to future changes).\n", - "from tensorflow.contrib.eager.python import tfe" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "# Step 2: Enable eager execution\n", - "\n", - "All future TensorFlow calls will execute the\n", - "underlying TensorFlow ops immediately:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "WPTUfGq6kJ5w" - }, - "outputs": [], - "source": [ - "tfe.enable_eager_execution()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "twBfWd5xyu_d" - }, - "source": [ - "# Step 3: Interactively Use TensorFlow!\n", - "\n", - "Now you can call TensorFlow functions and get results, immediately! No more `tf.Sessions`!\n", - "\n", - "TensorFlow will automatically wrap native Python types for you with operator overloading for TensorFlow Tensors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ngUe237Wt48W" - }, - "outputs": [], - "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", - "print(tf.encode_base64(\"hello world\"))\n", - "print(\"\")\n", - "\n", - "x = tf.constant(2)\n", - "y = tf.constant(3)\n", - "print(x * y + 1)\n", - "\n", - "# Most TensorFlow ops are directly usable with eager execution, giving\n", - "# results immediately.\n", - "print(tf.contrib.signal.hamming_window(x * y + 1))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "Numpy arrays are supported, too:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "lCUWzso6mbqR" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "ones = np.ones([3, 3])\n", - "\n", - "print(\"numpy 3x3 matrix of 1s:\")\n", - "print(ones)\n", - "print(\"\")\n", - "\n", - "print(\"Multiplied by 42:\")\n", - "print(tf.multiply(ones, 42))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBNP8yTRfu_X" - }, - "source": [ - "# Step 4: Define and Print TensorFlow Variables\n", - "\n", - "To define TensorFlow variables, use the `get_variable()` function as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "3Twf_Rw-gQFM" - }, - "outputs": [], - "source": [ - "x = tf.get_variable(name=\"x\", shape=[1], dtype=tf.float32, initializer=tf.zeros_initializer)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "45G7094TxsMb" - }, - "source": [ - "## Printing TensorFlow Variables" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "UJBJeZ5XxuwA" - }, - "outputs": [], - "source": [ - "# This does NOT print the Variable's actual value:\n", - "print(\"Printing a TensorFlow Variable:\")\n", - "print(x)\n", - "print(\"\")\n", - "\n", - "# A TensorFlow variable represents a reference to a tensor.\n", - "# The `read_value()` method provides access to the current value of the\n", - "# variable. Tensorflow Variables are automatically initialized according to the\n", - "# semantics defined in tf.get_variable().\n", - "print(\"Printing a TensorFlow Variable's value using .read_value():\")\n", - "print(x.read_value())\n", - "print(\"\")\n", - "\n", - "print(\"Printing a TensorFlow Variable's value using .read_value().numpy():\")\n", - "print(x.read_value().numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2njjWHcTpBEn" - }, - "source": [ - "## Changing a TensorFlow Variable's value\n", - "\n", - "To change a TensorFlow Variable's value, use its `.assign()` or `.assign_add()` method:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "v3wr6Erbo_hB" - }, - "outputs": [], - "source": [ - "x.assign(42)\n", - "print(x.read_value())\n", - "\n", - "x.assign_add(3)\n", - "print(x.read_value())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uhtynjHVpTB5" - }, - "source": [ - "## Use a Variable just like any other Tensor" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "7PbktdnHoehR" - }, - "outputs": [], - "source": [ - "print(x + 3)\n", - "\n", - "# This code will broadcast the value across the list of numbers:\n", - "print(x * [1, 2, 4])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GVChqwlwy1SI" - }, - "source": [ - "# Step 5: Debug Errors with Instant Feedback\n", - "\n", - "TensorFlow's eager execution helps you identify and debug runtime issues through interactive exploration of code snippets.\n", - "\n", - "Below, we'll define a length-4 vector, and attempt two `tf.slice()` operations,\n", - "one being legal and the other being illegal, leading to a runtime error that is\n", - "raised immediately." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "23ap04N0v4k0" - }, - "outputs": [], - "source": [ - "vector = tf.constant([10.0, 20.0, 30.0, 40.0])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "FCUMsIYxxRRa" - }, - "outputs": [], - "source": [ - "# Works, because the values of `begin` and `size` (the 2nd and 3rd input\n", - "# arguments) are within the bound of `vector`.\n", - "print(tf.slice(vector, [1], [3]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "T8me2oCNxpFp" - }, - "outputs": [], - "source": [ - "# The following does NOT work, because the value of `size` (the 3rd\n", - "# argument) causes the indices to go out of the bounds of `vector`. The\n", - "# error is raised immediately.\n", - "try:\n", - " print(tf.slice(vector, [1], [4]))\n", - "except tf.OpError as e:\n", - " print(\"Caught error: %s\" % e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "irxJhAgar84v" - }, - "source": [ - "# Step 6: Using the GPU\n", - "\n", - "You can place Tensors on the GPU by calling a Tensor's `.gpu()` method.\n", - "\n", - "The first operation executing on the GPU may be slow as TensorFlow initializes. Subsequent uses will be much faster." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "7J4N9baqaKCL" - }, - "outputs": [], - "source": [ - "# The example code from here on will work only if your notebook\n", - "# is running on a machine with a functional CUDA GPU. The following\n", - "# line checks that.\n", - "is_gpu_available = tfe.num_gpus() \u003e 0\n", - "\n", - "# Create some Tensors\n", - "SIZE = 1000\n", - "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", - "\n", - "if is_gpu_available:\n", - " gpu_tensor = cpu_tensor.gpu()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "4E-2n7VbzY1n" - }, - "outputs": [], - "source": [ - "# Time a CPU-based matrix multiplication\n", - "\n", - "print(\"Time to conduct matmul on CPU:\")\n", - "%time tf.matmul(cpu_tensor, cpu_tensor)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "vbSFW-T5zhZF" - }, - "outputs": [], - "source": [ - "# Time GPU-based matrix multiplications.\n", - "\n", - "if is_gpu_available:\n", - " # First use of the GPU will be slow:\n", - " print(\"Time to conduct first matmul on GPU:\")\n", - " %time tf.matmul(gpu_tensor, gpu_tensor)\n", - " print()\n", - "\n", - " # Subsequent uses are much faster:\n", - " print(\"Time to conduct second matmul on GPU:\")\n", - " %time tf.matmul(gpu_tensor, gpu_tensor)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "E5pIOe3Rz7iW" - }, - "outputs": [], - "source": [ - "# Second timing demo for GPUs, after it has been used once:\n", - "\n", - "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", - "print(\"Time to conduct CPU matmul:\")\n", - "%time tf.matmul(cpu_tensor, cpu_tensor)\n", - "print()\n", - "\n", - "if is_gpu_available:\n", - " gpu_tensor = cpu_tensor.gpu()\n", - " print(\"Time to conduct GPU matmul:\")\n", - " %time tf.matmul(gpu_tensor, gpu_tensor)" - ] - } - ], - "metadata": { - "colab": { - "default_view": {}, - "name": "Eager Execution Tutorial: Basics", - "provenance": [ - { - "file_id": "0B0kLcpwLFwKEVm9XNkFueGk4bTg", - "timestamp": 1504118841551 - } - ], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb deleted file mode 100644 index 5e0ec5cf8a..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb +++ /dev/null @@ -1,864 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "# Eager Execution: Working with Gradients\n", - "\n", - "This notebook demonstrates:\n", - "\n", - "* How to get gradients using TensorFlow's eager execution capabilities\n", - "* How to apply the gradients so you can update your variables" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "# Setup: Import eager and enable eager execution.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "OiMPZStlibBv" - }, - "outputs": [], - "source": [ - "# Import TensorFlow.\n", - "import tensorflow as tf\n", - "\n", - "# Import TensorFlow eager execution support (subject to future changes).\n", - "from tensorflow.contrib.eager.python import tfe\n", - "\n", - "# Enable eager execution.\n", - "tfe.enable_eager_execution()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "# Fitting a Simple Linear Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-39gouo7mtgu" - }, - "source": [ - "## Step 1: Synthesize some data\n", - "\n", - "To demonstrate fitting a model with TensorFlow's eager execution, we'll fit a linear model to some synthesized data (which includes some noise).\n", - "\n", - "In the code, we use the variable names `w` and `b` to represent the single weight and bias we'll use to fit our model." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "rQsdCg9PfIL-" - }, - "outputs": [], - "source": [ - "# The constants we'll try to fit our variables to:\n", - "true_w = 3\n", - "true_b = 2\n", - "\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "# Our inputs:\n", - "inputs = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", - "\n", - "# Our labels, with noise:\n", - "noise = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", - "labels = inputs * true_w + true_b + noise" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 360, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 127, - "status": "ok", - "timestamp": 1505502830690, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "O4lsC4ckAcar", - "outputId": "2f760690-cafb-4777-b970-91d839f99faf" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAFXCAYAAACC+2avAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt8VPWd99+TK7kykxtJQIebqZfaqogtrhKNa1ooEKl9\nCrpVn9ZNW6x9VWsbCi7aVUt01NZ9tq21KVZlFey2YkQNohhj3QWK2liCF5RIBCc3yEwmIZnMTOY8\nf/zmzJwzSSBAYibh+369eIU5c87vXLh8zvdu0TRNQxAEQRCEmCVurC9AEARBEISjI2ItCIIgCDGO\niLUgCIIgxDgi1oIgCIIQ44hYC4IgCEKMI2ItCIIgCDHOiIj16tWrufjii1m8eHF4269//Wvmz5/P\n0qVLWbp0Ka+//vpInEoQBEEQTjksI1Fn/eabb5KWlkZFRQWbN28GlFinpaXx7W9/+6QvUhAEQRBO\nZUbEsr7wwgvJzMwcsF36rQiCIAjCyTOqMesnn3ySsrIybr/9drq6ukbzVIIgCIIwYRk1sb722mt5\n5ZVXqK6uJicnh8rKytE6lSAIgiBMaEZNrLOysrBYLAB885vfZPfu3cc8RtzmgiAIgjCQhJFaKFpo\n29vbyc3NBeDll1+mqKjomGtYLBba2yeuuzw3N0Pubxwzke9vIt8byP2Nd06F+zsWIyLWt912Gzt3\n7sTtdnPZZZfxwx/+kJ07d/Lee+8RFxfH1KlTueuuu0biVIIgCIJwyjEiYv3ggw8O2Hb11VePxNKC\nIAiCcMojHcwEQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBOHXo6HCzcmUt\nTU2Z2O2dOBwl2GzWsb6smEfEWhAEQfjMWLmylurq6wAL9fUasJ6qqqVjfVkxj7jBBUEQhM+MpqZM\nwBL6ZAl9Fo6FiLUgCILwmWG3dwJa6JOG3e4Zy8sZN4gbXBAEQfjMcDhKgPWhmLUHh+Pysb6kcYGI\ntSAIgvCZYbNZJUZ9AogbXBAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFr\nQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhx\nRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBODE6OtysXFmL02mjsLADh6MEm806rGOamjKx2zuHdYww\n9oyIWK9evZrXXnuN7OxsNm/eDEBnZye33norn376KdOmTeOhhx4iIyNjJE4nCIIgACtX1lJdfR1g\nATRgPVVVS037RIuzz9dDTc33AQv19YMfI8QeI+IG//rXv866detM237/+98zb948XnrpJb70pS/x\nyCOPjMSpBEEQhBBNTZkooQawhD6b0QW9vv4qqquvZ/v27mMeI8QeIyLWF154IZmZ5j/wbdu2sXSp\neltbunQpr7zyykicShAEQQhht3eiLGoADbvdM2CfaEGH7GMeI8Qeoxaz7ujoICcnB4Dc3FxcLtdo\nnUoQBOGUxOEoAdaHYtYuHI7LAbPru61tD1AM2AAXkyY5sVr/CBxi3rwMHI5FY3cDwrCJuQSz3NyJ\nHdeW+xvfTOT7m8j3BhPz/uLi+klOTgQgOTmBnJwMsrIyuPnm5w2x7DKmTbuPgoJzaG7ew8GDt6PH\nuDMyNpKdncFNNz3Pxx+nM2NGFw8/vJCsrNhLOJuIf37Hw6iJdXZ2NocOHSInJ4f29naysrKGdVx7\ne9doXdKYk5ubIfc3jpnI9zeR7w0m7v2Vlz8XFuVduzT6+lSy2N69KRhd3zk5Z/LCC5dRWtrPwYOR\n7Xv3pnDjjYOvEUtM1D8/neG8iIxYnbWmaabPJSUlPPPMMwBs2rSJK664YqROJQiCIDB0gtlQsezB\ntg8nSU0Ye0bEsr7tttvYuXMnbrebyy67jB/+8Id897vf5Uc/+hF/+ctfKCws5D/+4z9G4lSCIAhC\nCLu9M1R+pdzauijrsWxVruUJx7JXrZrDrl2VuFzTsNkOsnr1EtaufWvQNYTYYkTE+sEHHxx0+2OP\nPTYSywuCIAiDMFSCmc1mHdSVXVn5Nk7nKsBCb6/G2rXrhxR2IbaIuQQzQRAEYXjoojxYTHewTmWD\nubyHEnYhthCxFgRBmIAYu5vpncrsdk1c3uMUEWtBEIQYYai+3SfSz3swK/rpp+cgLu/xiYi1IAhC\njDCYNVxVtXTI7UdjsOQzcXmPX0SsBUEQYoShyqhOpLxKEscmFiLWgiAIMcJQpVjm7S7a2t6ltJSw\nS3ywphojYUXLOM3YQcRaEAQhRhjKGjZub2t7F6dzFU6nconX1T1AaelU7r770mEL6XBF+ETc78Lo\nIGItCIIQIwxlDRu3l5aC0xlxibvdZ/KnPy06rjahwxVh6W4WO4xYu1FBEARh9IluGQpqPvXxCOlw\nRXg4IziFzwaxrAVBEMYRuku8ttaPx5MCLAQ0CgoODXuNoWLjQ51LktTGHhFrQRCEcYTuEr/hhv+i\npiYBeBY4xNtvu3G53ANiz4PFp4crwlLqFTuIWAuCIIxDmpsLgF5gOWChtVWjomJg7Hmo+LSI8PhC\nxFoQBGEcEG0hFxT4qK+fwrFiz5IkNjGQBDNBEITPiI4ON+Xlmygt3UZ5+TO4XO5hf69byPX1V1Fd\nfT0QoLBwN8dKAJMksYmBWNaCIAifEdEu6V27KqmtvS4cZz5aSVW0hdzcXEBt7SIqKgaOyDQiSWIT\nAxFrQRCEz4howXU6P09FRe2Qgmx0WRcUNFNf/xSQAXgoKPAcdUSmznCTxKRbWWwjbnBBEIRBOJbL\n+kQwu6RdwLts3Up4/aO7rBOBa4DFwLWhzyNHtJu9oqJ2RNcXTg6xrAVBEAZhNFptOhwl7NpVidP5\neeBdYCW9vRaqq9X6DkcJfX3r2LEjDjiMz5cWLsdqbs7B7AbPOalriUYS0WIbsawFQRAG4XjFaziW\nuM1mpbb2OsrK3KSkFA5Y32azkpychNv9bdzun1JTsyJs4UZb3QUFLeHzLVv21Elb/pKIFtuIZS0I\ngjAIw+3ypROxxDupr3+RurqXKS6OHxD71WPI5eXPhCxq8/pDvSREJ4r5fAkmy/94eoMb0WPV+/Yl\nUFhYSXZ2ETNn9kgiWowhYi0IgjAIx5tFHRHZGmABbvcWqqvT2LXrCWprrx+QrOVwlODzPcL27V1A\nNj5ffzhuPdhLQnSiWGnpNoZr+R8teczo7geNuXNlslYsImItCIIwCMfbajMisunAFvTOYk7n4kE7\ni9lsVpKSUnG7vwdYqKnRSEpaf9SXBKPotrXtAcoYjuV/PCVhEquOTUSsBUEQBmEoa3So7brI1tW1\n4HafyXAEcDChPNpLgtkKLqawsJK8vLMpKurl7ruHtvyPJsjH6+4XxgYRa0EQhEEYyhodarsusi6X\nm8svfwKnczFDCaAu+Pv3t6CSuoYnlGbRtZGXdzZbt15Bbm4GH3xwgPLyTYO6uo8myNI0ZXwgYi0I\nQswylo06IsLoBmrC9dD79iUQbaV2dLi55ZaXQiVXh5gzJ5kvfnEdzc052O0eVq26wCSkPl8PNTXf\nBzqBDVitXoqLE44plEcT3aO5uo8myDJZa3wgYi0IQswyGrXOwyXSMcwJ3Bauhy4srCTaGl65spYt\nW24Mb9u2bQNlZQG2br0CgPLyTab7sFofCO1rBa5l+vRnqaq6Ilz+NdTLiS66+/bF09HRRGNjEeXl\nz/Doo2VHdXWLII9/RKwFQYhZxjb5Se8Y9rzpGrKzi5g7V1mpBQUt+HwJvPZaErABWIgS4AyamvrD\nK0XfB2SjOphtAdJoa9uDyzXnmC8nuuhef/3TNDSswum0sHu3xo03PoHdjsSeJzAi1oIgxCxjmfwU\n6RjWhdGSnjmzJyygRotZ7fMgUAD00NbWTmkphnGWkTXmzQvyzjsP43SuwpgxPtyXE+Vuj+xXV6ex\nY8cVSOx54iJiLQhCzDKWyU+RF4WFDBVXHmgxfw5YRHLy7TidP8XptFFfr7Fgwe8oKzPex1dYtuwt\nnM7IsVu3gs023HKsQxhfIOCQuLonOCLWgiDELCMtQEdLWIv+bvXqOUReFAI4HFcOSG6LtvyhO/T7\n01Eu7nSgiwMHMnn11SVHPba3N5He3psoLKwkK6uIjo697Ntnp7z8mQGx63nz0qmp2YCawNXF/Pky\nHWuiI2ItCMIpw2Ax4fvuu5yVK2upqwvgdicDl1FfP5nBktmGEvTaWj8eTwrKCteAg4BqdgIaHR2V\nA65F9xps3Qq9vX7Uf8dv0NOTwFlnfUJDw3SczgwaGjz4fM/z+OPfCl8DJGG1eoGDzJuXwaOPXkN/\n/4BTCBMIEWtBEE4ZBosJR7fbhI3ANYPGi4dKALvhhv+ipkYD/gDk4PMlocqyAGpwuQoHWMjmHuEp\nqGQ2C273Il5//U7g1vA1bd/+AKCEuqRkfTjWDarrWVaWdch51sLEQMRaEIRThsES1gbGndOJjhfr\nFvXWrRj27WTz5oMUFf03gcCh0PbbAQuapqGywy3ActMYzGhr3eEooa7uZdzuyDX090+PuqZsQL0s\nqPGa0h70VEPEWhCEU4bBEtYqKl41CbjV+j7FxS5TIlnEot5AJLHrRYLBVSGR1YDHMQusDzWF+OjC\narNZ+fKX+9myJXINOTkHaWszZ4+D7hno5ni6ngkTAxFrQRBOGWw2azhG3dSUSUXFq1GJZB4cjuUD\nEsn27YtHucctwL1YLFPQNLMQQzvmDG0L8Klp2/vvv0lJyRFmzQqYXOIWSwD1IqASxs49N430dHP2\nOOiegSWha0mjsLABh+O6UXteQuwgYi0IQkwQnby1atUcKivfHvFWoyfSFa2jowmIxImTk9fg9Z6F\nWZyt6CIKO1BW9W3AfcDZwBG83ttoaNhCQ8P1pvM2NxcAV4XPd/jws2zYcMWA61Cegc2hZ+LG4bgO\nTYNlyzawd2/KZ96SVfjsELEWBCEmiBbRXbsqw4lUx9NqdLDyrNzcjPD3J9IVbfLkqTidG9FLsU47\nrZDZsz1s3/4A3d2ZBAKTQmumAW8CPwXeAGzAOcBiw2rpA8473OYvg5WyRbcy/SxbsgqfHSLWgiDE\nBNEi6nJN40QSqQaznJ999vrw98cSxsHEvrPzU4yW9ZEjlfzqV9excmUtjY2pHD78AR5PBt3de1Ad\nzGxEOp8ZO6C5gJ1AO3v2OLnhBicPPbT4pJq/yDzqUwMRa0EQYoJoEbXZDtLbG/mcn38oPOSioKAZ\nSAxNtTK7fo8lXqtWzWHnzntoa8sjPv4Q3d3puFzu8PGDiX12dpGp21h2dtGAkq/k5DXARcAelCir\nzmeZmR34fHfg9Z4P7AXuBiz4/VqosckLJCWlnrC7X+ZRnxqIWAuCMGocz4jLaOty9eolrF0b+ezz\n+amuVpOt1DSsaxisucn+/QHgSeBrwOQB4lVZ+TYtLbOAawgGLWzbplFREXEdDyb2M2d2snu3uT94\n9H59fRcBS1Au7/tISSmktBQcjjKWLXuL+vqrgM2mYyCD7ds/xe3+HsNxYw/2PB2OEpKTN4Zi1tIT\nfKIiYi0IwqhxPMlcg8Vjq6rs4d+Xlm4jInQZGEVv61bYtesJnM6bUC5oNYayuHjKAPFSmd1O1DSt\nLmAhdXUBGhubqKx8m/37W4gujVq1ag67dlXick3DZjvA6tVl3HnndswJZkfC1wNTuOyyI1RVqa5j\nEeu3K+qYLlQN9fDc2EM9z6efvkaaokxwRKwFQRg1oq3PfftSB8xr1jSGZX2b3b0ejKKn+mqvRu8+\nBhamTz+DqqqBGdXRmd2wAbd7Epde+if8/n9HdR4zD+6oqKgNJ7v19mosXVpJd7debuUDmoHvh86g\nAclApP+ncQ71oUP30NMzlbi4w8yblw4khLqfHduNLfHpUxcRa0EQRo3oeGpHx14aGswZ3sCg1uJQ\nfbhVL20f8ERo3URgAZFsbDia6EXHn5XYXoXfHwh9tqLizVU0NZ1BRcWrNDamYRTJSBexxSjX9lVA\nDSrT+wPgX2lufi18zqMNJHG53CQlDS+5TOLTpy4i1oIgjBrRceh9++wGoeykrq6Vvr4pKAt1IWAN\nW4tDuXxVL20Vu1ax6eXo4lVY2EBeXvCoojdz5pFQ/LkTeDG09QXgI4zdydzun1Bfr85dWLiWgS5v\njYgrezLqheFFIAd4gYKCwYV0sLjzcEutxnJkqDC2iFgLgjBqRFuU5eXP0NBgFkTzAI3lYWtxKJev\nUbCUIK4LZYV7cDiuO2YmtX58bW0LHs9PDedfh8redtHb68bvj8S0s7KmM3euOmdb27s4nStQrvh7\ngQySklbS359Mf/9cVDvQBcBfBj3/8cTxT0bYhYmFiLUgCJ8ZRqHdv99rGl6RkuKntHR92FocyuUb\n/QJgFLSKilePWfqkH19auo36eqM73A/00tX1KZr2C4wx7Vmz+sOu+VtvPURPzyaOHPkYv//HgA2f\nL5Kdrr94NDfnDCq2xxN3PpFua8LERMRaEITPDKPQKnd2RIxLSzEJ0VAu32gB7Orq5NVXf4guaD7f\nOh5/fNmAc+vH7dsXT0dHE93d8Zhd25OBa9G05zCKqdXqxeG4ElDiWVNzI2ZvwDVEZ6dDGna7e1Cx\ntdu1YcedJaFM0BGxFgRhTDhW/HWopKxoAUxMXItR0LZvjxtwzOHDxjnQG1HZ4CrrOzPTS3d3C8Hg\nTaG9zVOtiosThmy4EkloM2enT5q0i9Wrl/G9731EtNg+/XT04BBJKBOOjYi1IAhjwtEypI9GtGD2\n9+dgtpAPDzjmpptqQhncnahJWJF4dFzcM+Tnazidk0N7LwDuwGqdQXFxAqtWXRAuN2tr2wMUo2q5\nXUyatAuLxU1m5l407S7a2s5HDez4MZdc8is0bSrwGCpbXDVoOZ77loQyQUfEWhCEMWG43c2i9yso\n8Jmszby8Vlpa9PGSrfT2urDbN2GzHWDTpjJmzLDz8cdqAIfK1r4NYzwaDvPHP17OkiVr6OubgcXy\nMZdcks4f/nAlmgaXXfY4LS0/ALYA55KYuJaUlCx6erLwes8EvkZv72Ss1gdQHcwUfv+Foc9DN2g5\nFif6QiNMPESsBUEYE4abPBW934IFv+OKKx6hrs5CMHiY/v4errjiEIcPp/L++014vSo5rLfXRXHx\nL5k9+4vs21cPfA7owezG9jFvXjq//e1H9PWpnt2appGVtR5Ng5KS9bS0fAEl1KpEzO/vxu83J5Op\nuHU2Q3U0G6pBiyAMFxFrQRDGhOEmT0Xv19xcQFvbuwQCqrlKe7vGe+9VUl9/RSimq++7Ba/3Lhoa\nLMDVKFFNxyioU6Y0AqezdStE13qvXFkbcp13o4+1VEQnk6k1580LkpS0nrq6AG53K8aOZhJrFk6W\nURfrkpIS0tPTiYuLIyEhgT//+c+jfUpBEMaI4xncEZ08ZZyqZTx2sCSrDz4wj89U4zTBZjsQmtTV\nCfQxUFQvJTPzfk4/fSYdHXvp7k4JZXfrDVKeBRIpKPDQ1FRApGb6d6huZQNbnVqt71Nc7MLh+Ao2\nmxWXy80ttzzP9u1/ALKZNy+Iw/GVkXvIwinJqIu1xWJh/fr1TJ48+dg7C4IwrhnKtR0t4qtWzaG7\n20Ni4lr6+3PIzm7i7bcTaWubA3RTX78E2ExV1VJWrDiDmprb8fnsQCtvvHGE9HSLaXympn1ISclL\n+P3dJCSsIRBIAaZjdkt3A5NJTw8wa1ZPqO3p86HvazDXSa8LvSQsAZ4DJmOxrCEjYzpz5/aQlGRs\nxLLc9EJis1l5/PFvfRaPWziFGHWx1jSNYDA42qcRBCEGGMq1HS3iu3ZV4nTmoeK8GbS3dwA/wxgH\nbmrKZN++JhYufJ5gMNKk5PDhDcTH/4NJk9agaTPw+/fh9f6UhgYbEXd3MlBCxPX9D1QG94N0d/tp\nbEwNradPwUrH7GrPCZVYbWbfvgQ6OtxkZ5/HzJlHcDiWhsW5o8NNRcXwPAmCcDJ8Jpb1jTfeiMVi\nYdmyZXzzm98c7VMKgjBGDFUXHC3iym3dBhgbjAxsKnL11c8RDH4u6rsM+vtn0t9fTmFhJU7nl1FC\nrH/vB3YDS1HWsga8AawGLHg8GocP672+F6Ji1fuARabr1jOxy8s30dCwCqfTEuopHvEWqNptFdeu\nr19CX99fSE5OEvEWRpxRF+uNGzeSm5tLR0cH3/72t5k5cyYXXnjhaJ9WEIQxYKi64GgRV7HlwtBn\nN7AntIKKERcWNrBq1RIuvvgg8CEDZ0C3As/jdPaihHmx4ftEYAZKhDOIWM+6ld1FZmYuUGmYnnUd\ncB9wNoWFDTgc14Xv6WjeAn1spr7+jh1xuN3SHlQYeUZdrHNzcwHIysriyiuvZPfu3UcV69zcjNG+\npDFF7m98M5Hv70Tv7fBhNzfdVMPHH6czY0YXjz66hKwsszX56KNlrFixMbRPN2vXfotLLnmMlhYN\nFS+OuMCnTbuPd965iRUraggGVwGfALejYtDtKCs6H7gUaADsqIEaBSj394LQmkYSME7n6u6+j6lT\nz8XpXBzeIzW1kEWLjvDwwzeZrr+oqMf0olFU1EtubgZOp41ob4DF8qlpm9Np+8z+zkzkv5sw8e/v\nWIyqWPf29hIMBklLS6Onp4c33niDm2+++ajHtLd3jeYljSm5uRlyf+OYiXx/J3Nv5eXPhePRu3Zp\n9PUNZk3G8+tfLzJtOf/8PGpqNgD6HGkACzk5Z9LfH8/evSmh7XagAvgD8AVU/PkHRIu8Emz98/6o\n78wtSW222WRkfAw8hbK+D5OW9iF7987lO9+pNrmvf/zjL/DGG5W4XNOw2Q5w221ltLd3UVjYgdHi\nLyxs4ItftFJTY9zm+kz+zkzkv5twatzfsRhVsT506BA333wzFouF/v5+Fi9ezCWXXDKapxQEYYQY\nbhnWiQ6baG4uQLXhfAyj6L3zzm7OO28PZ51lrImeDEwFFpGfX09Ly2Sik8JU0xPlyk5IsBIIGL/L\nMp1j5swedu7sBH4Y3tbevoH29qvCCXB5eWdjt3fi8/nD7u7eXo21a9dTVWUfxOWvXOdJSdIeVBh5\nRlWsTzvtNKqrq0fzFIIgjBLD7TA2VFLZYOValZVvG9qGHgkd14MxvqxpBTidNxIM3kNZ2XoaG1Np\nb3+Pnh6NuLg/cs45mZx//jq2b+/A7Y4khcFeVCMSK3APA/uFb8Bq9VJcnIDDcTnnnVdLdOKa/nun\n8/M4nUuor9ewWv/IYC8jQ7UClRi1MBpIBzNBEAZluBbzUEllt9zyElu2qGzv+nqNF164g0DgrvDn\nBQvWsWDBOmpqfIbzgJpkZaGz024Yp9kTfnHYts1Fbu6DdHUB3IPFkk1c3Kf09/8EJdQagUAPkYSy\nbiwWK0uWBHA4rgx7B1SSmwvVSjQNleR2KcqKj7QKhUMYhV+6kQljwcBZcoIgCCiLWYkUgMb+/R9S\nXv4MLpfbtJ/NZuW++y7Hbvewb18ql1/+BCUlz7FtWwuqMxiAhUDAjlH8X3stgaSkROLjm4GvEmnr\n+S7gQtP2hs9lfnF4hvb2NPr7LwJmoWnX0N9fBNRgtT5KYWElKhltOSpLfDmTJ3cDsGzZW+F72LSp\njEmTfhnabwnwMzIz/5NJk+5AJao9BbiYNy+DsrL1nHfes5SVrT8h13ZHh5vy8k2Ulm4b9BkKwrEQ\ny1oQhEFxOEro61vHK68ECAS6cLu7qK6eyvbtf+Svf/22KX5tdJmDhtO5EZXBvQG4FiX6jRgt1N7e\nZKqrl2Ox3INxUIYS2Pvwem+jokJ1MTO72l1EN1BRMenFTJv2JJ98YkH913Ynykp2091dSHV1PHBZ\nKCb9MHl5ZzNp0gy83sgLRFzcJLzefwuvXVhYyUMPXXfStdLDDSkIwlCIWAuCMCg2m5Xk5CQCAWPj\nko20ta3hllseISkpNRx/bmxMY2AfbjXVCjaj6qKTgcdR86SnAN9ATbmaiu76jhxfAPyOLVuslJc/\nw+rVc9Bd7Q0NGVHJY16U21qjo6MJj2cFSvwvBHYBd4X214UdnE7V5ASexBzbzjZdR17e2SPS1ORE\nk/AEQUfEWhCEIYkWGV2Et2/vwu3+HrqlOGXKHShhzkANuuhFiV8Lqq92I5oWaRmqLG5QrmY/8L+Y\nG5skAT+jr+9RqqsTqav7G8XF8Tz99Bxuuul5tm0zCmwLaWk9/PM/r6exsQin02ilw8Dr7yISz+4h\nM/NeZs48C7vdg8/Xbyq9Gqn49FBJeIIwXESsBeEURs/YdjptFBZ2DCjPihYZFVceaIEePpyAcRBG\nQsJdBAIbUNnZk5k82YXbbRRNN/BrlKtcubaTk9fg988kGExBNTbRXd7fwe22UF2t3MdJSQA/B+ag\nLOrvEx9fFWoN+gy7dycSEWM9acyGPiHL6+3E6707fK3p6ZVs3apmTbtc7lEpvRoqCU8QhouItSCc\nwkTHmqNjqQ5HCUeOPMJrr2kEAk7i4rK45JKH2Lv3CGoaVTdwMYFAPkbxPuusc5g5s4emptcGtVhV\n1na84RgbfX3TgY+BuahxlQuAHAa6jzNRLvUl4evs6ckIX++WLY/Q16eL8SLgDmy2WcyfH4fDsZxv\nfGMnu3dH1szOLgqvM1Q51skyWusKpw4i1oJwCjBUg5NjxVJtNitPPfUvpm3l5ZtoabkFXXgtltvR\ntHOIxH5d7NnzNh99VITNtodHHilD0+CVV+7E75+NillfS2Lievx+o4A3Ar8wrZuRkYHHE+0+1qiv\nbzadr7+8jtPXAAAgAElEQVT/AKWl27DbO5k58wzee89oxV/A7NkJVFVdBsDMmUdCAzkiDVIEIdYR\nsRaEcc5wOo0NlY18tFjqcAVe07JQFnEVqnd3F8FgJb29quPX0qWVzJ07Db//34kI8wbmz8/kf/7n\nDrzeuSh39udN606ePJudO6/kllseYfv2LiAbn6+fn/98Hps3dxAMRlzdmvYL6uvVveXn/wJz0th7\nfPRRIeXlz+BwlIhLWhiXiFgLwjgnWojr6h6guDjPJNpDWdC6cKmYtcskXMMVeNU0pNLw+bemc7lc\nBQPOHxfnYc8eNxbLLJQrfSHK9R1Zd9IkJwBJSanhZLaaGo2kpPV85Svp1NQsN5wzsnZPTz6RjmgN\nwApcLls45l1VtXTYLunhtlwVhNFGxFoQxjnRQuh2n0l19SKM8eehLGg9ljrYoITIum6ghq1bCZdR\n9fWtY8eOOI4c2Y/ff6bp/Kq1Z+RcmtaI3T47dP5O4EWCwSAtLbOAr6FqoTcCC4iLu51g8MvAEVpa\nfkBFxeZBXzSefnoO77xTidM5DeVWj2SS9/a2ohLXQL0IbEHPAt+3L/64BFjqo4VYQcRaEMY5g2ds\nm+PPJ+L6LShopr7+KVRJViK9vUuorp7MSy+t4Z/+yYbb/R3gEeAAZrezF2OrT6/XRm1tVyi2nQXc\nZth3I3ANKSl9XHbZ0/zP/2Tg8fhQXczcvPiik/nzC03rt7Q0cMstzbhc01D/hX0/tE4asAO/f7ph\n//0YG6h89NEdfPnLTtzunzAcAZb6aCFWELEWhHGOLsR1dQHc7kkol7Kyns1WpMbTT885DjduIsZy\nLF1Yvd6LeP31N1EW60qUtfwEaiBHO6qLsdFFvQGP59rQPtEzoPuA5wgEGnj11UmGLO6rgY34/d+n\noeEOpky5k9bWTCCHlpZp1NT0oCzq7xPp7f0ukAt8E3gUVfaVazqf13s+Xm8iwxVgqY8WYgURa0EY\n5+iubJfLTUVFbbhcyuG4nIqKod24RiEvKurh7rsvNQl5c7O5bEpZyhpwhP5+O5GuY1ZUE5PridRG\n3wecg5o9nQc0AR+iyq6Mk7KSgCX4/Xpf8IENWDyeWSQnt2O2yO8AZgPVqBeEbuAW8vP/MzQ+MxX4\nDip2bbT6+1CW//AEWJLRhFhBxFoQJgjRtbwdHW7q6gIMZUVGx2O3blWJafooy/37WzAL3QcoUfwq\nmnY/0EYkVmxsF2pDCfWi0P7LUeJ9F8oK3xD6eRi4OXSMGo9pPp9qwKJp+4AZmIXc+HKgERdXyeLF\nm1m9+uusXbuerVuht9eC8jJsJDXVj9V6EKdzRegY87jM4T5TQRgrRKwFYYKycmUtbncyRgFsa3sX\nl2vOoCVYbvdsqqt7eeGFZwkEfoBqevI4CQkHiY930dcHyjLdiKaBGtDxBMpSNSd5KYu6m0gnsjwi\nVvi1obUnh36BalGqhFUJ//+GjrkTTZvOpEmfmu7DYslG0yLXnpmZHxbVqio75eXPhLK/rcByFi3a\nyN13XxdOWLPbzeMyBSHWEbEWhAmKEuPLiCR7fYDTuWKISVYaKuY7hUAgDlV+dROwhUDgCwQC/wvM\nAv7VsP8TKAs3A9iHxXIP8fF5BAKTQts04K+AhylTPqa11Xiut4F+LJZ7yMgoJDHxQw4f/hg4HdUi\nVG9peit9fRZaWlwUFlaSl3c2druH7m6LqT/4vHlB071Hu68ffngJ/f3xYiUL4xYRa0GYYOix6P37\nA8ALRMqjGgAL+/bFU16+iX37EigsrKS7Ow+PJxXoR8V6VwHPM3Bs5YOYXdFeIq7opSQn34HFkkIg\ncD1qulYkOe3ccx8hGFxDe7sVSEFlmF+IpnnxeBaQmPhbIuVWoCzvFoyu9by8s009vCsqjLHkr5ie\nQbT7OitrYGmaIIwnRKwFYYKgi3RdXWu4NElZqA8CU1GZ0y/S0dFEQ8Oq8PcLFqwjI8PCn/6Ui7KI\nLaj4cXTCVzbmmHIbcC9wJtCL16u3En0KJeSRY1991UJcXAoqSWwjymqPZJn7/YVRax9BxbUHTwST\nWLJwqiFiLQgThEjC2POYRfZzKMsYrFYv2dlFoVnO6vvm5hxefPEqtmypxOPJRAnkQuBhzHHoD4B7\nUOVQn6Cs9dNQ/43obvR7UWIcB/wB1VAlm2CwjWBwNsYs78j1pQE+EhPvxO+/ECXUXwX+jB7DLixs\nwOG4bljPYbCmJ7m5GcN/kIIQg4hYC8I4xihMjY3NKGs0Oqv6g9C2i0lNbaGpqdXwvYuWlgYuuiie\n1FQfHs8hIsM0koG1wNmo+dR+4N8M696DuQ57PxExVo1U4EbD9/eGfkZf35vArcyf/0fee68Bl6uA\nYPBBEhPzSEhwMW9eBg89dJ0pGexoXcgG6zr27LPXj+RjF4TPHBFrQRjHDBxxuQFlFW/AYulE0yaj\nksImAz/B6ZyDsnZ19/X7tLTcTkuLGiepYtjxeDyRrl9qNvVUVDmWbhF3hn4+jxLfhahY9FMoV/jn\nQvsaLegzQ9fXgYpPzwQ+4owzTufsszfj82XidN4aPm9f30ZgOe+8U3nU+46uH5euY8JEJG6sL0AQ\nhBMnWpiURfssYCEpCVSZlJVIzPkaVLz4Z6i4snnSVV7e2cTFTQltawLuIxA4DTW+8gPUCwGooRv/\nhnKTXwO8SE5OV+j35aiMbo9hfw34O6rL2b+EzvuvQCVnn51OVdXSIZqwdOJ0JvGlL71MefkzuFzu\nQe/bKMh2ux7rVueVrmPCREAsa0EYx+Tnt2N2KX+KsliXk529FqfT+F02A8XQQ3QS1/79h4hY6SsN\nx98R+mVHZY4b1+rC69Vbieq11A+jeocfRjVKuZWMjN/j9f4Wv/8H4WN1oR28x/mLwG243Zbw1Kyf\n/ewC3n//TZStoWq5jYIsXceEiYiItSCMA4aK0VosAZRL+xxUYtZNJCb+ioUL13PTTZdTVnYHXu8Z\nKBHXk8f0lqC7gHxgLSkpUykuDuDz+QkGe1FCrTcyIfTzDOA6VH11E+aXhAz6+oyNS14GvoDKLs9A\nxbxtBAIF5OYewOmcjHLHv0hjo5fzzvt/ZGbmUlhYSUdHPl7vx8BZKE+B2YK++urn8HrvDp970qQ7\ncDi+G35WkikuTERErAVhHDBUjLa5uQBlyR5BWco1zJ49i6qqpZSXb8LrvQtdnJOSKklIuJ2enjhg\nGiqurGqws7PvIzm5kOrqG9HHWMI+zILsDH13AGVdr0G9JAAsJCnpMIHAGjStCJVsdhvKotbLxzR6\nexPp7b2JwsJKenoScbt/gsdjwePRcDo3AuUUFq7F6bwRlQneHzpeXdP+/V48Hv2zcu9bLGdIJzJh\nwiNiLQjjgH374ol0IusKfdZdx06MYyA7OytDx6QSsUq34PPdh893H2bX9qNAKocPF1BX10JEBK8F\nqlCZ4fkogc5CDc643XD8htC+GoFAK5p2t+E7NaVLfc5An1kNVvLyzgagvn7g4I7U1Azi4n5PMHgm\nyoJ/mMREF37/atzugee12Q6OwBMWhNhGxFoQxgEdHU2ozmJKrDo6lCA7HCVs2/Ys3d0RIXc6M7nh\nho20tzehRk0aB20UYnZtu4Dv0NtrobdXL69agcoeTw/9Mo67/H3U8T5SUp6gtBS2bp0V9V1a6Pca\neXlO2toy0NuPFhR4SEpKHSRGrXHwYDvB4C8M2+8jIeE0/P7I2gkJXSQmPoHNdpBNm5aMwBMWhNhG\nxFoQxgHRjUy6u/MpLd2G3d5JWlob3d03YxS3mpofkJFRScQa34PK3Nbjyrqr20qk3MsKTCUh4X7i\n4vz4fBehksOMAtwedbwPTfuE1auXs2tXdUjw9VjyLs48Mxjq5Z3Ntm3Gmux14USwxsZUDh/eS1aW\nnVmz1g8i+oXYbAdMa3/taykSlxZOKUSsBWEcMHPmEXbvjoiVxzOJ+vqrqK/XyMy8n4Edyyx0dWWh\nOoFtQcWoVwE5KDd2GrAas8t6OZBIIHAPSsD/GZXR/RyRCVr5qASzT9AbpHi9LoqLf8mMGbPp6FiD\nxTILm62ZTZuWMWOGHYDS0m2ma2xuzglN7oL4+ATmzp2KwzEfm83Keef9P5Mwx8W9z2OPLeI3v5EM\nb+HURcRaEGIUYwZ4QcERFixYR3NzDvv3f4jbXR7ay0JcXA4DO5Y9h7Ki70fVNH+CyuZuAaaj/ukb\nBb6XSExZjzHXYIyFq45lFtRkrFzD8Vvweu/ivffUfmVl66mq+qHpXqLLsvLzD1FSsh6n8/NAN/X1\nSwA1DWzTpjKKi+/A650LHCEY/Cm/+c1msaSFUxoRa0GIUaIzwBcseCRUB52NcZqWGg+5jr/+tZfu\n7k+Bi1CW8OmoxiMbMVvRG1ATuIwCvxeoNHzuIjLUg9DPPOC7od8/aTg+zbSfPtXLWGbmcJTQ17eO\nHTvigMP8/e+dtLYas8U3huutZ8ywc+aZc0ICrpAuZMKpjoi1IMQI0bXU+/aZrd/t27twu7+HLqiZ\nmfeTnh7gwAE7s2YFuPRSqKkxCq4+0jJ6cEYGkEpy8hr6+opQJVnXAveRklLIxRd3smePO9yCNLJe\nu2Gdr6EsbTvwIcaBH8apXvX1Gjt33oPXO4nu7kwCgRRUh7PJRJLZrEAaBQVt4WcRbYlLFzLhVEfE\nWhDGEKNAt7Xtwem8CbBRX69RWFiJ2fo1dyCLi8vB6VyK07mFhgYbCQnNmEVZH2kZPTiji4yMOI4c\nyUEN2zgHZWmfTmlpAJhMS8vNqCSyDajGJMmobmcuVAw8DeU670MN67gPOJ1Jk97D5TIniLW0nAbc\nYDi/XtJ1DsrVvhyVABeplT6RLmRHG+4hCOMdEWtBGEPMgzjK0OueIZ3u7ngWLPgdzc0F2O0efL5+\namoiohsMtqISwFTcNxBIxizKHwLrAB8JCXeQmmqnt7cVvz+Prq6bQsdGyrL0TmDLlr2FuW3oY6hE\ntXZUDFwvq1qMst5/HfrcH2rCsiHqOj7GPPAjncjMaj9KvFfQ3Pxa+LmcSBeyW255iS1b1JSv+noN\nn28djz++7LjWEIRYRcRaEMaQgYM4VN0zWPB4FvHOO5U888ylVFa+zYEDqRQWVpKdXcTMmT3s2NGD\nx6N3KFPlUCrTuwg4hEokawb6uPLKqTz++DJKS7dRX38VqtXnFNO5LZaZVFS8SkGBL6r+OQnV4/t7\nqKYo0Znnt6EEWo9xL0QJsB/1wvBjIrHpDSi3ezfqBaAGZWWfvKtbxcONYQOZUyRMHESsBWEM0F22\n+/cHUMlaKlksISGDQCAiOE7n5/n615/D6VyFXtvc3d3K4cOddHbOwFwjnQccxOxyvg/4N/7+919Q\nWrqNtrY9QDHKlW22xHt7J1Fd/VVycx8gLq6SYPBclKguBJ4hMfGXTJ6sceiQUcg7iMTBdXe7FWWx\nbwQuQAm1up+MjB4uuSSN5uYUCgr+Avhpbn52hMqx9AEk+rUdPsn1BCF2ELEWhDEgeg611foAxcVT\n8PniTK5u2EN7+ySU8H0K3IbHsxGP5ybMMWA97mu2llUi1ye0tEyipUUD4oiLewzoIRj8FuamKWnA\nb2lvn40S/UuIWMQp+P134fGsJGJFd6GsZz0uvjD0nQc129ofWidyPxkZbTz00HWjEkueNy+dmprI\ntc2blz7i5xCEsULEWhDGgGj39/TpZ1BVdQUul5va2kiNMXyf/v77gVtRcd9OlGgbY8CHgZ8Dp6Hi\nwy4iItsM/BfG0q1g8FGUqNeiXNyXoBLMsgFzJzRlraeg11/7fJ9HxbHdKBd2H6rZyixUL3Er8+f3\n8NFHHQZvQCRJzelcQUXF6NRMP/TQYpKSamlq6sduD+BwLBrxcwjCWCFiLQhjwGClSR0dbm699QW8\n3lTgXVQf799hsVhD+3Whz3c210w7iSR96f29P48S4FuBNxgqLq72vxPlEg9gdqufTWLiLvx+Y1xc\nb1eqZ3FvBCJWfn7+PVRV/V+WLXsr1B5VT1J7At3CHq2aaRmNKUxkRKwF4SQYrFxI0zhmCdGqVXPY\ntasSl2saNtsBVq8uY+XKWmpqMlGCGBHI/v5VKKH7J1Ss2Si8XSir1rgtK7R/AcrCPow5lpsVtX8i\ng7ce3cP8+Vbee68y1GnsCPA14uJuJxiczWA13IcO5bFs2VuG2Lhu4SeG1tyA3R44mUcuCKckItaC\ncBJEdxnbseNOLJZEWlq+SHQbTSOVlW+H3MRq2tXatetDFmc8oAshqCztIpYsWU9dXStudwCz8LpQ\n7m/jtkmoGHRC6LNuMetx5vej9j8Ns3gfAdYQF5cNTGLTpq+wdu3bNDVl8v77/43X+wsi5VnmGu5A\noIv6+u8BZRQWqpeR3t5EdDe61erF4bhyJB69IJxSiFgLwkkQHXtubc3E7KbeyL598dxww5Ns394F\nZDNvXj8HD9pMx+lWeH19AsqtHRHA5OSPqaqqoKTkJdzuuURiyZ+gBnTEoVzZXyAu7m0SE0/Dau0h\nP9/PO++sQpVyAVyKcksfQrnN1QuFwije7cDdBIMWtm1TLxL6y4YqrzKWZx1Cud3PRDVJ0T0IFvLy\nzmbu3E6qqyO13MXFCdKoRBBOABFrQTgJomPPaqqV0UpN49Chf9DQMBNVp2yhpkajsHAtRoFsa3uX\nRx5ZwvPPr6e/H2ANMAP4kOeeUz2yOzo+QM2n/hmq3KsIVaOs1igsrKS2dkVYDE8/3YHRnR5xbx9C\nJY3prURdpKTcyec+d0FoSMh0ol8kdDIzP6S39ymUlR4EWiksTCMvz0Jb236czhWhPbVQOdbxdyIT\nBGEgItaCcBI4HCXs2mWM6YJRhAsLG+juzid6KEZW1nSCwXtCrTgP4XTm8POf/5WMjM/hdn8ntJ+b\nxMTf8KMfHaCx8QX6+rJRTU+CqCEdlqg1i/jRj14KNQc5hNc7jYHu7TtQGdr5JCSsITGxCJvtIK+/\nfiOZmVmUl3dSXR003YOxWcm5506ltdU8lzovL4etW6/A5ZpDRcVmkzBL0pcgjAwi1oJwEthsVmpr\nr6OiQh9l6QbWceCAlY6OvWRl2Wlvfx9lyUYEsKOjiba2eIwNTF555U5SUuyoEqgkQMPvn857730F\n+CbKMs4Grg8d86RpzY8+eoeGBqMlvQqze/sQytJ+ArievLxK6uuVkObmZtDe3oXDUUJX1yb++te1\n9PfnkJfXyurVXw/fb0tLtOcgKyzmgwmz9OsWhJFBxFo45TlZQRlMpMrLN9HQsCpUvuQCfoXqo51D\ncvJenM6fAq9hFD6//zz8/q8DT2F0b0cGX6SjMrvNk6+s1qmkprbgdJ6FWUhPJ9L0pBs1IUvPFrdw\n6JCV8877T7KzizjrLB93330pNpuVjAwrfv8PUUM4VMz6vvsms3JlLR988AnGF4BJk/6Ow/HdIZ9N\ndAIerBdLWxBOABFr4ZTnZAVlMLGPTjxLTEwmISEPm+0AaWmz+fBDGwOzsveimo34MIuuPviiG5X8\npR8zGYsFtmy5iNLSF4nUQOvrdaJGUBpFXwM+ADz4fB04nbfjdFrYvVtj69YHKC7OGzCas6kp0/CM\nzE1OZs8+86gvNtHPQeZSC8KJIWItnPKcrKAMJvYFBUeor9cTsRrw+1fj96syrbi421GiOR01ZcuF\nSkzTUIMyEjGKrsXyDzTtDSAXaMNYhlVSMpnKyrfxeH6KEtInAC8WSxslJalYLI/wt78l0Nv7CX7/\n5NCx/4pqQ/p703273WdSXb1owGhOu91jeEZ6k5PNwCJmzVp/1Gcjc6kFYWQQsRZOeU5WUAYT+4IC\nH2ZXduT7YLAIZeUeRDUNKUSJbyJKjL9NxH39Ppr2A5S4bgD+D/AUcXFTyM9vZu3aMr73vY9QQl2D\ncnHXk5CQQnp6DqtWzaGy8m2ami6goaGVQOBaw5V7MFvi3YCFtjYbmZn3Ehc3hXnzgjgcX6Gi4lXT\nM7Ja36e42HXM7G7JBheEkUHEWjjlOVlBGUzsm5qMiVjdmEVxHzAXZSk3oTK0dSt6DZo2GX1spGpu\nYkW5x53AfwM/Ixi04HSqeLLdrlFf/yKq8cgW4Iv4/X+jujqVHTueprVVTzp7LOo6JqFqtnWL/VpU\nYxM3Hk8u8G2SktZjs1kHeUbLhxXXl2xwQRgZRKyFU56TFZTBxN5siS5g0iR9OEcD5vnOD2O0ujVt\nKuaksNND3+k9wfVhHjVAOi+++AlPPTWX6upGlFDrDUgWAxtob3cb1r8KPclNZZtnYh7c8SAwFfg+\najZ2JCQgoisIY8uoi/Xrr7/O2rVr0TSNq6++mu9+d+jMUUGINYzJY0VFPeGM6ejv7HaNp5+eg6ZB\nRUUtH3zQR1LSSgKByWhaFgkJCeTkvMGhQzMwzneO7tsdH3+Q/v7lKOFNA/4GrEe5rI3DPJSL3e9f\nxHXX3YHqIKYnkaWH9rMQF5dNMOgCnkPVWbtRZWT7UF3QjIlsn0OJPOgxdIkxC0JsMKpiHQwGufvu\nu3nsscfIy8vjG9/4BldccQWzZs0azdMKwogRnTzW1xfJFDd/52LXrofp6cnH7U5GtQA9D11Uu7s1\nurvvZaBLvBdju87MzG5crl+i3OTdKGv6d8TFHSYYfCp0nC7cABb6+magyrgexNyx7A4CgWyUq/si\nlBv9bsP390ZdS1doTY3MzGYuv3y9xJgFIUYYVbH+xz/+gd1uZ+rUqQB87WtfY9u2bSLWwrjhaJni\n5u+2hAdzRFzKUzBbrlNR85/10qckoAKYTGbm/Vx+eT61tfmodqLGcqtzCAZ3EElYM2drJyU10tc3\nGbgg6nznh873o9Dn56K+n0JcXCWZmflcfHEQTfPT3PxsyJX/LWleIggxxKiKdWtrKwUFBeHPU6ZM\nYffu3aN5SkE4LvQZ0sYhGw899NWwUB0tU9z8XRpmIcxhYLb1p6h48JbQfsbM7CyqqpYye/bTUesk\nomZbzyYya/paVO/wmUAj55+vMWXKeurqWnC7jefrwzzCMtqqn0QwuBq3WyM9fSO//vWiE3+QgiCM\nKqMq1pqmjebygnDSRGZIR4ZsJCVFXN3G5LGiol7uvjviFjZ+19a2B6fzUpT1qgGfEB9/iJSU/Xi9\nOaSmdjB3bhJJSX/hwAErDQ31GIWzp6cJgNTUZjweo6C+jZqQFT2M42z07O3333+A555bSmNjE5dd\npiey7UG9GNQYzrMAWE1CwukEgx0Egz8I3YmFl1/uw+VyizUtCDHKqIp1fn4+Tqcz/Lm1tZW8vLyj\nHpObmzGalzTmyP3FFk6nMdlL/Xz5Zbj55s08/PBCiopO49lnrx/02I8//pitWz/C651OUpKb5OT7\n6euLCGt//4P097t5//2vMmuWPXzcsmUbaGiwY8z61rQscnMzyM+fTUuLMRu8ELOl3YNyg98U3max\n5JKbm8HNN+8OCfUSYD5KqA9jjImXlc3i2Wf/lWXLnuJPf5ocWkPD5UpizZo3ePrpa07mccY04+3v\n5vEi9zexGVWxPvfcc/nkk0/49NNPyc3N5YUXXuCXv/zlUY9pb+866vfjGX1YwkTls7y/kRoQUVjY\ngbI8jVauxp/+dA11dXdywQWn09ycg93eyaOPltHfHx8+trj4v/F6VUJXX9/AMiz4HL29izjnnDWc\nddaF4evcuzcF1RAl0go0Le1+PvjgAG1tjcBqIpb0PZhd1wdRLvGI0H75ywHa27tC6+qubiuwHKv1\nAdzuVeFrfuGFRzj33Cc57bROMjPvx+M5K3TMQvbufW3C/v2Uf3vjm1Ph/o7FqIp1fHw8a9as4Tvf\n+Q6apvGNb3xDksuEESE6S9vne4SkpNTjFm+Ho4QdO35Pa2ukhSf4AQutrZnU1NwYPseKFea4rsrC\nNoqzLvzmjmB9fRdRX78k3IpUNTHRO5KloHqEZ1BS8gRO57+gLO40EhPfRNM6CQSM1xYAesOJYfPm\nBXnooa8Aegx9Sfj4KVPexGJJQLnmu4EFBAIZNDRYaGhYQWHhWjyeReHrLShoobx8k0zIEoQYZNTr\nrOfPn8/8+fNH+zTCKUZ0lvb27V243SrufDzDOGw2K7m5X6S19RuGrZtRYmseB/nxx+mmYxMT38Pn\n08up9gNWLJZVaNoUVCb4wtA6R8JrNDVl8vTTc/D5nmf79k85csSH378aj8cSilXrE7bgnHOCfPCB\nJ6pF6BPAdSxePPD+VAxdnyftxuc7Pfyyoa7j58A09KSzrKzpzJ0bicd3dSXIhCxBiFGkg5kwLonO\n0lZzno8+jGMo13lHxweYLeJ/oCxRv2n71KkdpvXmzTuNurprUAKryq1UUuUToWP+GlrrJlQzkhfZ\nv99LRcWr3HnnpVRWvs3WreD361neVlRWOeiZ521tB+jtjVxDYuJHLFwYqX8+WjigtHQbZsv/QmAR\nqu5aY9as/rAY5+ZmcP75zx7zGQqCMDaIWAvjkugWnz5fPzU1Rx/GMdQozKys6TidxqQuG5BGUtLf\n8fkiLmhN8wMRgfzb3zKJuLKNomhDJXlp5OfXc/75f2H7dhdu909wuy1UV2vs2lUZVZetZ3nvITPz\nAOnpnTQ2FnHWWekEg/fQ2WnHZjvIpk3fZMaMSLLa0cZ75ucbx2lG3PLJyZlkZ1fS2FhEefkzOBwl\n5OZmyIQsQYhhRKyFcUl0r2qXy01SkhLv/PxD+Hx+Skqeo6OjiezsImbOPGKY0+wGati6FcrLn+G0\n047Q0BA993k+gUAzxlpop3MzYBbIwTuBvQtYsFrfp67u/2KzWSkt3UZ9fUTQW1ryMQu8L3TeFfT2\n/gaPZzVOp1qvrGxod/TRmrZYLAHMDViUWz47243TuSo8xxrW8+yz18uELEGIYUSshQmBUbzLyzdR\nXX0jSvwiohSZ01wDLKe3V1m5Cxaso6xsPXV1AdzuScDngQcJBmcCT6JaeU5mxoxuYKBAKsv7DlQ8\nuAOYDnhITvawbNlb2O2dFBT4TFZrMNiIWeCdwCpAw++fynDd0UezhpubC1DDO9TLSUrKc5SWQmNj\nUagP95kAAB2VSURBVOhFwLy+DOsQhNhFxFqYcETE1Ni9y0J2dhFz565n61bo7Y1s37LFD8SRlLSP\nSy9NYceO9/H7jT221wCn8cYbbXz8cdMQ8fJ/Ae4HvoxyN/8Tra1NtLbGU1+vYbM1oAR9BtCIGk/5\nKOACckhI8HHmmU9y8KATtzsXo5C3tb2Ly6WGhETHp49mDUeuU5VxlZYqC728/JmQRS3ubkEYL4hY\nCxOOiEh1YU4QcwNJJCe3mJK21Pzoa+nr0/jb39bQ3z8Ts+V8EbAEp1Nj6dJKamuvo69vHVu39hMM\ndqFqnp/D3GnsTuDfw59drmYiPb9dwC9DP28DLAQCGtOmraOjw4/bnYkS9rMAC07nCioqlAteud87\nqa9/kc2bXyQ//xCbNpWZ4tg6Qwm5uLsFYfwhYi1MOHQx2rcvno6OylDMugefzx9yj3cCG7Bavbjd\nzcC3ULHddPr6JqHKsIyWc6T0yuWahs1mJTk5iWBQj1u7gD9hFvjZUZ+Nru0tqOlYz5v22bEjLtTA\nxAIsxVjGFXGFW1Bu/GsIBi3hF4j6+h8OeA5DubXF3S0I44+4sb4AQTgROjrclJdvorR0G+Xlz+By\nucPf6WL05z/PZ+7cacTHJwAaBw7o7nErcC3Tp2eRnNwN/A8qE/tS1HCM6cDtKOt3FZAMPAW4sNkO\nAkZXuxvVuSwdJewQGdox1Gd96EdX1D6HMQu8uYzLbu8M7Wd277tc007gCQqCMJ4Qy1oYM06mZejR\nSpaG2ieSYBaJ1WZm5vL66z6MFmvEoq4M/VKfU1LuZNOmbwJGV3sNKiFtPpFe3/XAdeidxPLz/8E5\n56Tw1lsPEAza6OvbT1/fYlR2trLwi4sT8PnSTOVn8CbgJjHxQ1avXoamESr5ysKY+Ka/QAiCMHER\nsRbGjOEI7lAcrWRpqH2ys4v4whfWsWNHHHAYny8Nl+t0Iv20zRYrmMurZs/+AmvXvk1T00cUFPhY\nsOB3vPZaGr293ai49TWhdeopK3s93EnM4bjB9BLicrmpqNBjxgEcjiux2azh8jOVld4K3ArY8Ps1\n1q5dD2CqzY6LqyQ/HzZtWjKsZyYIwvhFxFoYM4YjuDC4BR6dkd3W9i6NjbOprHw7FKtuort7CkYL\n9PDhvRw4kIjb/RNAjcMsLFwL5KFi1p+iOnzplu1HGC3xDz98h927fwxsob5+Cvn573DxxbBt23J0\nK1qNpnRTVXXLkPetu+n1+9LLuxyOEqqqluJyufnSl17G7Y5MBDPHrNXPL3zhbLZuveL4HrogCOMS\nEWthzBhux6zBLHCHoyTkEv48cASncwVf//rDIctT1Vfr61qtD5Ca6sfpXAG8gVHwsrKm09PTh9t9\nLSr+vBHoBeJJT7fS3R3pbOb1TkElhy0HOmlp6aalxQU4UAlk76EakMwIdwY7mls/+r5eeukOzjjj\ni8yceYR587yDdGTTpMOYIJyiiFgLY8ZwSog6OtzU1bWiMqe7gIXU1QUAyMs7G6cz4gJWiVadKAs5\nsj9kk5WVHJpdbSznctHR0YT6Z2BM9Ipj0qQP+dKXckNWs3EQhje0dgORUiy9i9mtqBj2tVRXR9z6\nQ8Xmoz0LXu9cdu9ewu7dkUYtA5+NlFwJwqmIiLUwZgynhGjlytqw21qJ4gbc7klUVNSGRk1GLE2b\n7QC9vS+i1y4b909N3R/6HEnqSk1tCVnincCDoTOqY71eDYvlEcrK9CYqicBpgHGKlTG+fQ7K6s4I\nb9Nd10PF5gc2V4mUiG3fHsfOnZcPsMyl5EoQTk2kdEuIaQa29vQBC2lqysThKKGsbD3nnfcsZWXr\n2bSpDKvVO+j+2dlFoX1fo6wswM6dV5KXdzaRUq5CoMh07JtvJlFVtZTSUg3l+p5i+F5PSoOI0Kah\nLHe1TXUecw8am+/ocOPz9ZCYeCeqocq9wFfDx+ovJIIgCCCWtRCj6K7j/ftbMDcoSQYmY7d7BrXM\ni4vfCrmgzfvPnNkzYF+zZbsA1S50cfjYI0eacbnchvi4RiQBbQEqLn4xSqi/Sn7+b9C0Plpbn0OP\no1dUbB7gAbDbPaxcWUtNzfdRVv2LZGZm0Nv7K/z+81Gu9oU0Nb02sg9VEIRxi4i1EJNEXMeq21hm\nppf09BaysuzMmrWeVasuoLx804A4sB4Hb2xM5fDhvUPuv2LFGezc+Qlxcb9H09rQtGyU5RwZien3\n9/GlL71McXE8mzYtYfHi/6Kt7V5UMtmnXHppOllZ7tCam3E4bmDZsrdobdXj6CrePm1aIYWFkU5q\nDsflLFv2FsYGLTNnPovdnkF19VVE9wQfbu25IAgTFxFrIWYwJmIpi7oTo5ht3fp/wvuqyVqROPCO\nHXdywQX/v717D66yvvM4/s4dSAI5QIBEuiGAEay2TC11YVxCsY0SwKBopXWkRZuV0sEx7Qw3124t\n3VBTrbZDhyJip1AqWNYkUAhVA4RWKcvWTTEqZYg0CLmS5DQJhlzI2T8eTs41yUlyDufJyef1jyR5\n8jy/x4if/G7f379QVTWelBQb+/bdicVyj5frjbra+/f/DZttKvZtXUbFskSMFd23AGeBWVitVyks\nXAgcYO7cmRQUrMAepnFxO/rorR/qPsMabMye7VhwVlv7IcYsVAuw8PqCMc8V7mvXHtA8tYgorMU8\nPM+Jfg3jPGnPbUru88A1NaMpKjIWf5WW2igpeZ709AleVl4bVcpsNvszXgVGYdTyjsGo8x2O8yEc\nsIeKitFERUW4PLOqarzHO2zYcAenTm2msXEyHR2tdHZ67iNft+6oS3GT5OTN5OU9isWS4LHCvbfj\nMUVk+NACMzEN9wBOSLjavXjMfZuSo0421/853uV7rdYZFBau6F6k1VNdbSOclwMP4Dhw4zxGr95+\nTSy1tR9y7twZl2c6/wJhr1V+773/Q2VlCq2t99HZGef1evf3nDDh1u6hbvf30l5qEQH1rMVE3Lcy\n/eu/dhET00RFxWjWrj3iUmTEfcjY4LywrAXn3qx9LrukpBqr1blKWTze64I7evUjRpyisvJx4G3g\nBSIj4/nqVyPIy3MMs3uOCuwBFpGQ8DyTJ6fS0HCW8vIUsrPfICmpvcfiJjq+UkS8UViLaTiOthxF\nQ8NZ3n13NE1NI4H5lJaOwbl2uMWSwNGjj/LUUwc5caKZrq6RjBr1X3z6adL178nEOQjtK8dd63I3\n0dLSRXGxZ4979OirhIe/CtTT1TWRq1dPYN9j3dlp429/20xj4z9Zu9Y+x96Ja489DhhDevpE4FPK\nyjZQWRlGWZmNhQt/1UPBEx1fKSLeKazFNOxBlZ2dT1mZY07Xfq6z+/ytxZJAdPQorNYngDCamowg\njI6OoqLimNeeqc3m8hG5uf9Gbu4ujh6toqnJ0eP+9NOP6ezcdP3j3TiOtQQIo7LyNpYuzae6ehoQ\nAdTg3LNPSDhDenqj28pv43urqpJU01tE+kVhLUHR2/GYnoVQjLlfb/O37td6C0LnZ9XWfkBl5SPA\nCUpLLZw6Vcirr36ZoqIyjCpmxtx3Z2eM030XERb2PDabYw82NFJdzfW2NQNfJyrqP/nsZ79w/ZeE\n5S7z0KrpLSKDobCWoOjteEz3cHPupdo5iqZ0Ar/AmKOezJkzZzl/fjqpqSlenwVZwHPAOowe8hKW\nLv0B7e3P4dqTHwf8DmNOu4nY2Gu0tPwEo6zoFYzKaP/h8j2xsVO89pg1Dy0ig6WwlqDo7XhMz3Bb\n7lIYpKHByoIFu64vLmsB/oF9q9XVqzbuv38zpaVrenyWUVrU8XFbW6rb12MxVoR/B8ee6k20tKzC\nqP8dS2Rkk8u2LIhlzhz7QjdXmocWkcFSWEvA+XIetfPQcF/h5r5PGTbjHLaNjZNdnlldfRqoAyYB\nTURHv097u+PZMTEfc/Wq4+Pw8L8QE3Mzra2OeyYm3sq8eYc5e3YkKSlW2tvDXY6wTE4u46WXHvXr\nvyNVLhMRO4W1BFxP51E79557Kh/qjWtP+Z/ANYzDMIxqYBbLRS9D369h1P22MW9eM7Gxjmd/97uZ\nfOtbRiETi+Ui+fnfIDfXtcb41KmfsnfvCurqjIM6GhutREc79/4fHVS49jYtICKisJaA8zbk7d57\ndi8f2ltYTZpUh2Pl9SGc545HjPgB+fkP88QT53Ad2o4HrEAR77wziowMG3v3Oupul5be7vKMvDxj\nq1hP88z+HtrubVpAREQVzCTgfKnK5R5Wb74J2dlv0Nho7b7GXiXs3XerMHrKBzAWejm+b8aMO0hN\nTfFS4awZo/DJclpbV7hUN3O/f0ZG8fUiLF9mz547AFi27CSf+cxmFizY79Euf1DlMhHpjXrWEnB9\nrYb2drBFa2sUhYXLgV0899yXWbfuKCUlnVitMcDNGNXGwFix7Riurq39kIwMSEq6wsKFO6iqGk9S\n0mWgg2PHYl3mod17r84nfZWWHqKk5C1Gjap2mR+/eHEPZWUr8PcwtVaMi0hvFNYyIN4WRCUmxvf6\n9Z7mdD0XjD0HrMIeqJ6lPH+CUdP7MAAjRjzDzTfPor7+LJWV36Gy0kJpqY2srF28+ebd3W2Jiemk\ntXU39pO2ej4cxCg9arWGYbXux3PPt/+HqbViXER6o7CWAfG2IMo4PrLnr/cURp5bq27FOBrTGA72\n/PoM4JcYx1oa27UmT95BRMStVFZauq9zPuXKOewTEp4nPX2i18NBjLY6lx5twbPmuIapReTGUljL\ngHhbEFVfbyU7e7+X86h774m6b+OaNOk0V69eBuppb48lKanNrUjKOVpaEl32OZ84EU56uvftYO5t\nnTLlZrZv77l4iethHwtJTt7MuHFpNDaeIyHhM0yb5jgFTFuuRORGUFjLgHjbJ716dZHP51E7y8tb\nQFvbDv7yl3CgHputDat1JRBGUZG3gy+Wc+edr2G1Ovd468nLM+a43ed9fS332dNhH/ZtWYmJD3Zv\n3bLTlisRuREU1jIg3vZJL1z4v7ifRz1lSkGfC6YslgRiYqKxWpdgzEN3YAR9JpDgtd73nDlxFBW9\nhrElq5k5c+KwWBJYv/4LLFu2n7//PYk//nEbqak3M2WKY7GZL4u3+jN/rC1XInIjKKxlQLztk25s\njMJ5fjc9PdLrcLOd8xCyMWy+H1iBo7e8B1jutSf80ktLiI4+SkXFNVJSOsnLWwzAsmX7XRarffTR\nHj76aEX3YrPBcB7m96USm4iIvyisxS+MHuV8jIAdQVTU/1FefgvZ2W/0OI9rDCHbe9MzgCqce6kj\nR3aQkbGrx+pm3nq/jY2TXe7hz9XbzsP8PVVi05YrEQkEhbX4hdHDHIOx//l3dHQ8S1lZGGVlrvO4\nnr3p/wYex3FutKOXmpFB9/nWvs4LWyyf0NoamNXb5887rxL3XolNRCQQFNbiF3l5CwgL28mxY9do\nauqgq8v7PK7nnukXcD43Oioql8jIz2CxXGTjxvsAKC8fhXNIfvzxKI/n238JGDNmOg0Nz2CzJREW\nVk1q6nTS0nb5pcebmtrMqVMa8haRG09hLT5raLDy1FN/vL5q+zJz5sTx0ktLsFgSsFgSiI6Oxmpd\njrE4zHuouS/IioyMp7PTfu0YOjpS6ej4Bq2tNnJzd7F9ewoNDX93uV99/VngHpe2uf4S8DWysnax\nffsK/Gnr1kza2jTkLSI3nsJafLZu3VEOH7YPWdsoKnqN6Oij3cPAjmHiTGDP9TlnXELNfUHWqFEN\nxMUZ+5g/+eQ8Vms29gM39u/v5NSpXxAbm4QxFx4HtDB2bIpH227EquyxYzXkLSLBobAWn3lWEoun\nouJa99cdw8QJwHIyMjznlh2FRzqxWkfQ1PQdmprGMHv2LqZOnUBh4Rjsq8BttjAqK22MGPEMsAl7\nwE+btsujbVqVLSKhTGEtPjMC0V6TOxb4gKQkxypvX4aJ7QuyMjKKKS1d2v35iorR7N17B7CLwsJm\nHD3pZq5ds7gVRfG8r1Zli0goU1iLz/LyFnDy5C+prjZqcsMSYEf31/szTOzeE66t/ZCHH4aUFBsx\nMVW0ta3u/lpExA/Yvv3fe72fVmWLSChTWIvPLJYEJk26jepqx1B4VdX4Ad3LuSdcW/uhy2lZ8fHb\naWtzPCM19TZ/NF9EZMhSWIvPvJ07PdC5YeeecEYGLqdlRURYcV79nZbWNui2i4gMZQpr8Zn7udPJ\nyZvJy3t00Pd1HxKfMyee6GjNP4uI2CmsxWfuq8EnTLgViyWhuyBJZaWF5OSGfh8T6bk4bLGOmRQR\ncaKwHqYGcg5zT9ujPKuS9e+YyIEsDtM50iIynCish6mBnMPc0/aoG3VMpHNA19Z+QGXlasCic6RF\nJOQprIcJ957oxx/H0t+A7akH3FOP29+9X9cefBbGXuyv+9x+EZGhSmE9TLj3pJOTc+mpfndf3EN4\n40ajmIkxZ93Y3eMeSO+9N54V1GKv/1kVy0QktAUsrLds2cLrr7/OuHHjAMjJyWHevHmBepz0wT3o\nxo6dwuzZA1tx3VMIJybGU1fX3OMzB9v7de/BJyeXMWFCl1aMi0jIC2jPeuXKlaxcuTKQjxAfuQfd\ntGnXBtzL9TWE+1uvu69hc88580e1qExEhoWAhrXNZgvk7aUf/Fk729cQ7u8z+xo2V0lRERmuAhrW\nu3fvprCwkNtuu43169cTHx8fyMdJL/wZdL6GcH+feaNWlYuIDDVhtkF0f1euXMnly5c9Pp+Tk8Os\nWbOwWCyEhYXx4osvUldXR25u7qAaKwNXX29l9eoizp+PIzW1ma1bMxk71lxDyA8//Dtef91Y3Q02\nvva1Pezd+/VgN0tEJOgGFda+unTpEqtWreLAgQN9Xuu8QCnUuC/AupGys/NdCpdkZfl/X/Jg36+x\n0cratUddeuxmmpMO5s8v0EL53UDvN9QNh/frS8CGwevq6khMTATgrbfeIi0tLVCPEh84hpitQBFv\nvgnZ2W+YqvKX5qRFRLwLWFj/9Kc/5aOPPiI8PJybbrqJH/3oR4F6lPjAsSisCFhOa2sYhYUD3/vs\nbeW2L78diohI/wUsrPPy8gJ1axmADRvu4NSpzVRVTcJmG/wiLm8rtwsKVviruSIi4kQVzIaJzZvf\nu3685Wv0VrnM3mMuL4+goaGCcePSmDr1isdwuVZui4jcOArrYcIRrpnAHkaO7CAjA49tV44e8x5g\nA5WVYbz/vudwube91vX1VrKz9+skLBERP1NYDxOOcE0AlpOR4Qhf5/nnf/yjEyOA43DuOZeXjyI7\nO9+jHrh95faGDV9g1qxfcfHiOvpbC1zHXYqI9E5hPUz0VsjE9TSr3RjD5M04D5dfvnyGsrKnsQdx\ne/sOfvObh7vvkZ2dz8WLtzKQoXF/H/ghIhJqFNbDRG/bolznnxeRkPA8kycn09Cw+fqc9accPZqA\ncxCfOBHu5R4tDOQkL81/i4j0TmEtbvPPY0hPn8j27fe5XJOWthXnIIZ6L/e4D2OuO5bk5DLy8h4d\nwPN13KWIiDuFtfhU63vOnDiKil4D4oFm5syJ87hHTMxhzp4dSUqKtV8nYvnzkBERkVB0Q8qN9keo\nl5Qbqu/nSynQofx+vgjl9wvldwO931A3HN6vL+pZB0ioVfhSKVARkeBRWAeIKnyJiIi/hPd9iQyE\nVjiLiIi/KKwDJCXlnxirpkErnEVEZDA0DB4gWuGsymQiIv6isA6Q/i7ICsVgU2UyERH/UFibRCgG\nm+btRUT8Q3PWJhGKwaZ5exER/1DP2iRCseSm5u1FRPxDYW0SoRhsKqQiIuIfCmuTULCJiEhPNGct\nIiJicgprERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgpr\nERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NY\ni4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NYi4iImJzC\nWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicoMK68OHD7N48WJmzpzJBx984PK1\nbdu2kZGRwcKFC/nzn/88qEaKiIgMZ4MK67S0NLZs2cLs2bNdPl9eXk5RURGHDh1i+/btPPvss9hs\ntkE1VEREZLgaVFhPnTqVKVOmeARxcXExmZmZREZGMnnyZFJSUjh9+vSgGioiIjJcBWTOuqamhqSk\npO6PJ06cSE1NTSAeJSIiEvIi+7pg5cqVXL582ePzOTk5LFiwwOv3eBvyDgsLG0DzREREpM+w/vWv\nf93vm06aNImqqqruj6urq5kwYYJP35uYGN/v5w0ler+hLZTfL5TfDfR+Q12ov19f/DYM7tybXrBg\nAYcOHaK9vZ1PPvmECxcu8LnPfc5fjxIRERlWwmyDWKb99ttvs2nTJhobGxk9ejQzZszglVdeAYyt\nW/v27SMyMpKnn36au+66y2+NFhERGU4GFdYiIiISeKpgJiIiYnIKaxEREZNTWIuIiJicacN6x44d\nzJgxA6vVGuym+NXPf/5z7rvvPpYuXcrjjz9OXV1dsJvkV3l5eSxcuJCsrCzWrFlDS0tLsJvkN73V\nwh/Kjh8/zr333ss999zDyy+/HOzm+NXGjRuZO3cuS5YsCXZTAqK6upoVK1aQmZnJkiVL2LlzZ7Cb\n5Dft7e089NBDLF26lCVLlrBly5ZgNykgurq6uP/++1m1alWv15kyrKurq3n33XdJTk4OdlP87tvf\n/jb79++noKCA+fPnh9x/gHfddRcHDx6ksLCQlJQUtm3bFuwm+U1PtfCHsq6uLjZt2sSOHTv4wx/+\nwMGDBykvLw92s/zmgQceYMeOHcFuRsBERESwYcMGDh06xJ49e9i9e3fI/Pyio6PZuXMnBQUFFBQU\ncPz48ZAsW71z506mTZvW53WmDOvc3FzWrl0b7GYERGxsbPefW1tbCQ835Y9gwObOndv9TrNmzaK6\nujrILfKfnmrhD2WnT58mJSWFm266iaioKBYtWkRxcXGwm+U3X/ziFxk9enSwmxEwiYmJzJw5EzD+\n3zJt2jRqa2uD3Cr/GTlyJGD0sjs7O4PcGv+rrq6mpKSEhx56qM9r+6xgdqMdOXKEpKQkbrnllmA3\nJWBefPFFCgsLiY+PD6lhK3f79u1j0aJFwW6G9MJbHf/3338/iC2Sgbp48SJnzpwJqQJUXV1dPPDA\nA1y4cIFHHnkkpN4NHB3T5ubmPq8NSlj3VG/8qaeeYtu2bbz66qvdnxuKvZi+6qnn5OSQk5PDyy+/\nzG9/+1vWrFkThFYOnC/14rdu3UpUVNSQmyscSC38oWwo/v0ST1euXOHJJ59k48aNLqN3Q114eDgF\nBQW0tLSwevVqzp07x/Tp04PdLL84duwY48ePZ+bMmZw8ebLP64MS1j3VGz979iyXLl0iKysLm81G\nTU0Ny5Yt4/e//z3jxo27wa0cOF/rqS9evJgnnnhiyIV1X++Xn59PSUnJkBw1GEgt/KFs0qRJVFZW\ndn9cU1Pjcx1/MYfOzk6efPJJsrKy+MpXvhLs5gREXFwcX/rSl/jTn/4UMmH93nvvceTIEUpKSmhr\na+PKlSusXbuWvLw8r9ebasI0LS2Nd955h+LiYo4cOcLEiRPJz88fUkHdl4qKiu4/FxcXM3Xq1CC2\nxv+OHz/OK6+8wtatW4mOjg52cwImVHqkt99+OxcuXODSpUu0t7dz8OBB7r777mA3y69C5WfVk40b\nNzJ9+nS++c1vBrspftXQ0NA9PHz16lVOnDgRUv+//N73vsexY8coLi7mZz/7GXfeeWePQQ0mnLN2\nFhYWFnJ/0V544QXOnz9PeHg4ycnJPPvss8Fukl/9+Mc/pqOjg8ceewyAz3/+8/zwhz8MbqP8xLkW\n/qpVq1xq4Q9VERERPPPMMzz22GPYbDYefPBBn1amDhXf//73OXnyJFarlfnz57NmzRqWLVsW7Gb5\nzV//+lcOHDhAWloaS5cuJSwsjJycHObNmxfspg1aXV0d69evp6uri66uLjIzM0lPTw92s4JGtcFF\nRERMzlTD4CIiIuJJYS0iImJyCmsRERGTU1iLiIiYnMJaRETE5BTWIiIiJqewFhERMTmFtYiIiMn9\nPyQ+uNKCpR6MAAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0xa813090\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# Plot the Data (Optional)\n", - "\n", - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs.numpy(), labels.numpy())\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JaFHyAG9nDET" - }, - "source": [ - "## Step 2: Define our TensorFlow variables\n", - "\n", - "We'll use Keras's object-oriented [`Dense`](https://www.tensorflow.org/api_docs/python/tf/contrib/keras/layers/Dense) layer to create our variables. In this case, we'll create a `Dense` layer with a single weight and bias.\n", - "\n", - "(**Note**: We're using the implementation of `Dense` found in `tf.layers.Dense` though the documentation link is for `tf.contrib.keras.layers.Dense`. When TensorFlow 1.4 is released, the documentation will also be in `tf.layers.Dense`) " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 34, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 22, - "status": "ok", - "timestamp": 1505502830753, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "z9r-ZeyrXu3A", - "outputId": "6230a7a3-29fe-4d08-f101-da80425bad82" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 4, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# Create TensorFlow Variables using Keras's Dense layer.\n", - "\n", - "wb = tf.layers.Dense(units=1, use_bias=True)\n", - "\n", - "# We can access the underlying TensorFlow variables using wb.variables.\n", - "# However, the variables won't exist until the dimensions of the input\n", - "# tensors are known. Once the dimensions of the input tensors are known,\n", - "# Keras can create and initialize the variables. Until then, Keras will\n", - "# report the variables as an empty list: [].\n", - "\n", - "wb.variables" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "docKLUaonYG_" - }, - "source": [ - "## Step 3: Define our loss function\n", - "\n", - "Our loss function is the standard L2 loss (where we reduce the loss to its mean across its inputs)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "0_w8ZJSCtuY7" - }, - "outputs": [], - "source": [ - "def loss_fn(inputs, labels, wb):\n", - " \"\"\"Calculates the mean L2 loss for our linear model.\"\"\"\n", - " predictions = wb(inputs)\n", - " return tf.reduce_mean(tf.square(predictions - labels))" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 34, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 24, - "status": "ok", - "timestamp": 1505502830875, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "RkNbXoXkpjVH", - "outputId": "c36fc98d-3a57-4074-901d-c10ae017ae3f" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\u003ctf.Tensor: id=40, shape=(), dtype=float32, numpy=7.3549819\u003e" - ] - }, - "execution_count": 6, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# Test loss function (optional).\n", - "\n", - "loss_fn(inputs, labels, wb)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 51, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 57, - "status": "ok", - "timestamp": 1505502830981, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "K_7beXoHOU7t", - "outputId": "1ad0856a-02ec-4117-a6c0-b41030981d87" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "w: tf.Tensor([[ 1.56891453]], shape=(1, 1), dtype=float32)\n", - "b: tf.Tensor([ 0.], shape=(1,), dtype=float32)\n" - ] - } - ], - "source": [ - "# At this point, the variables exist, and can now be queried:\n", - "\n", - "w, b = wb.variables\n", - "print(\"w: \" + str(w.read_value()))\n", - "print(\"b: \" + str(b.read_value()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YIlebeb_qYtC" - }, - "source": [ - "## Step 4: Create our gradients function using `implicit_value_and_gradients()`\n", - "\n", - "With a loss function defined, we can calculate gradients and apply them to our variables to update them.\n", - "\n", - "To calculate the gradients, we wrap our loss function using the `implicit_value_and_gradients()` function.\n", - "\n", - "`implicit_value_and_gradients()` returns a function that accepts the same inputs as the function passed in, and returns a tuple consisting of:\n", - "\n", - "1. the value returned by the function passed in (in this case, the loss calculated by `calculate_linear_model_loss()`), and\n", - "1. a list of tuples consisting of:\n", - " 1. The value of the gradient (a `tf.Tensor`) with respect to a given variable\n", - " 1. The corresponding variable (`tf.Variable`)\n", - "\n", - "Test it out below to get a feel for what it does. Notice how the first value of the returned tuple (the loss) is the same as the value returned in the cell above that tests our loss function." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "v1spZQ4NwW1U" - }, - "outputs": [], - "source": [ - "# Produce our gradients function. See description above for details about\n", - "# the returned function's signature.\n", - "\n", - "value_and_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 153, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 46, - "status": "ok", - "timestamp": 1505502831114, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "21WMcpsmFFLd", - "outputId": "f51b3171-33f5-4f87-8bf7-0be2dc8edc8a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Outputs of value_and_gradients_fn:\n", - "Loss: tf.Tensor(7.35498, shape=(), dtype=float32)\n", - "\n", - "Gradient: tf.Tensor([[-3.00773573]], shape=(1, 1), dtype=float32)\n", - "Variable: \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e\n", - "\n", - "Gradient: tf.Tensor([-4.06519032], shape=(1,), dtype=float32)\n", - "Variable: \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e\n" - ] - } - ], - "source": [ - "# Show outputs of value_and_gradients_fn.\n", - "\n", - "print(\"Outputs of value_and_gradients_fn:\")\n", - "\n", - "value, grads_and_vars = value_and_gradients_fn(inputs, labels, wb)\n", - "\n", - "print('Loss: {}'.format(value))\n", - "for (grad, var) in grads_and_vars:\n", - " print(\"\")\n", - " print('Gradient: {}\\nVariable: {}'.format(grad, var))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JVDWpL9VYWdP" - }, - "source": [ - "## Step 5: Create an optimizer\n", - "\n", - "We'll use a `GradientDescentOptimizer` to fit our model." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "DudNEebMKDWN" - }, - "outputs": [], - "source": [ - "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YBeJYxY8YaiO" - }, - "source": [ - "### Step 5a: Test Our Optimizer\n", - "\n", - "Now we have everything needed to start fitting our variables to the data!\n", - "\n", - "In the next cell, we'll demo these capabilities. We'll:\n", - "\n", - "1. Print the current values of `w` and `b`\n", - "1. Calculate the loss and gradients\n", - "1. Apply the gradients\n", - "1. Print out the new values of `w` and `b`\n", - "\n", - "You can run the cell multiple times. Each time, you should see the values of `w` and `b` get closer to their true values of 3 and 2." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 102, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 103, - "status": "ok", - "timestamp": 1505502831285, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "diDZfrMJM3OC", - "outputId": "d585fff0-ecb3-4e98-9b33-bbae07a95d8c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Values of w, b, BEFORE applying gradients:\n", - "(array([[ 1.56891453]], dtype=float32), array([ 0.], dtype=float32))\n", - "()\n", - "Values of w, b, AFTER applying gradients:\n", - "(array([[ 1.86968815]], dtype=float32), array([ 0.40651903], dtype=float32))\n" - ] - } - ], - "source": [ - "# Test the optimizer.\n", - "\n", - "print(\"Values of w, b, BEFORE applying gradients:\")\n", - "w, b = wb.variables\n", - "print(w.read_value().numpy(), b.read_value().numpy())\n", - "print()\n", - "\n", - "# Calculate the gradients:\n", - "empirical_loss, gradients_and_variables = value_and_gradients_fn(\n", - " inputs, labels, wb)\n", - "optimizer.apply_gradients(gradients_and_variables)\n", - "\n", - "print(\"Values of w, b, AFTER applying gradients:\")\n", - "print(w.read_value().numpy(), b.read_value().numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "61TgeLVlKEQp" - }, - "source": [ - "## Step 6: Create a training loop\n", - "\n", - "Of course, now we can simply turn all of this code into a self-standing training loop. We'll also capture our loss and approximations of `w` and `b` and plot them over time." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 397, - "output_extras": [ - { - "item_id": 1 - }, - { - "item_id": 2 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 225, - "status": "ok", - "timestamp": 1505502831550, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "VukGe-huNaJ4", - "outputId": "f0a8d665-1910-477c-d8ab-c94ccdc4afcd" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[2.111051321029663, 2.3047544956207275, 2.4602210521698, 2.5850086212158203, 2.6851789951324463, 2.7655951976776123, 2.830157995223999, 2.8819968700408936, 2.9236228466033936, 2.9570505619049072]\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAFXCAYAAADnFpTQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFUbBfAzu+m9koSShBQCSC+igIAgRRGkChJEiggo\nHURAEBQBQeADRcWCha50ULFLk6IivYRQQwskhPS6O/P9sckmm4Rkk2x2difn9zz7bLuZvC8JHO7M\n7FxBkiQJREREVOlUchdARERUVTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMjArdlJQU\njB8/Hk8//TS6d++OkydPVnZdREREiiMY8znd6dOno2XLlujbty80Gg0yMzPh4uJijvqIiIgUo9TQ\nTU1NRa9evfDbb7+ZqyYiIiJFKnX38s2bN+Hp6YkZM2agd+/emD17NjIzM81RGxERkaKUGroajQbn\nzp3DoEGDsH37djg4OOCzzz4zR21ERESKUmro+vv7w9/fHw0bNgQAdO3aFefOnSvxa3g5ZyIioqJs\nShvg4+ODgIAAXL16FbVr18aRI0cQGhpa4tcIgoC4uBSTFSkHX19Xq+8BUEYfSugBYB+WRAk9AMro\nQwk9ALo+jFFq6ALArFmzMHXqVGg0GtSqVQsLFy6sUHFERERVkVGhW7duXWzdurWyayEiIlI0XpGK\niIjITBi6REREZsLQJSIiMhOGLhERkZkwdImIiMyEoUtERCbRuXM7uUuweAxdIiIyCUEQ5C7B4hn1\nOV0iIqKy+OijFTh69BAEQYUhQ4ajU6fOuH8/HnPmzER6ehq0Wi2mTJmOJ59sgwUL3kZU1HkAArp3\n74nnn39B7vIrDUOXiEhh5s6dhd27d5h0mz169MLcue8aNXbv3t9x+XI01qz5Fg8eJODll4egadNm\n+PXXn9Cq1eN48cVhkCQJmZmZOH/+POLi7uGbbzYBANLSUk1at6Xh7mUiIjKp06dP4qmnugIAPD29\n0LRpc5w/fw716j2CH37Yha+++hyXLkXD0dERtWrVwp07t7F8+RIcPXoYTk7OMldfuTjTJSJSmLlz\n3zV6VloZCq80l/e8ceOm+Oijz3H48EEsWDAXAwcOxuDBA/D11xtx9Ohh7Ny5DX/88StmzHhLjrLN\ngjNdIiIyifxwbYbff/8VoijiwYMHOHXqBOrXfwSxsbHw8PDEs8/2wrPP9sLFixeQmJgIUdSiffsn\n8fLLoxEdHSVzF5WLM10iIjKJvLOX27d/EmfPnsbQoS9AEFR49dXx8PT0wp4932PjxrWwsbGBk5Mz\nZs16G7GxsXj99TcgSSIEQcDo0eNk7qJyCVIlrThv7esjKmmNR2vvQwk9AOzDkiihB0AZfSihB8D4\n9XS5e5mIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIismjHjx/DmTOn9M93\n7NiKn3/+0STbXrv2K5Nsx1gMXSIismjHjx/D6dP5odurV1907fqMSba9Zo15Q5dXpCIiogrbsGEN\n7O3t0bfvAHzwwVJcvnwJK1Z8gmPH/sGPP+7C7NnzDMZHRV3Ahx8ug0aTDWdnN7z55hx4eXlj8+ZN\n2LlzG2xsbBAcXBujR4/Fzp1boVbb4Ndf92DixNfx779/w8nJCQMHDsa4caNQp04ETp48gczMTMya\nNRdr136FK1cuo2PHzhg5cgwAYMaMqYiLu4fs7Cz07/8CevTohVWrViI7OwvDh0eidu0QzJ49D7/8\nsgebN2+CVqtB/foNMGXKdJOuE8zQJSJSGOe5s2Bv4qX9snr0QloJiyg0btwM3367Hn37DkBU1AXk\n5ORAq9Xi1KkTaNy4mcFYjUaD5csX4733liEsrBY2bdqGTz/9CDNmvIX167/Bli27YWNjg7S0VDg7\nu+C55/rqQxYA/v33b4Pt2dra4Ysv1mDz5k2YPn0KvvpqPVxcXDFgQC8MGBAJNzc3zJw5B66ursjK\nysLIkUPQvn1HjB49Ftu2bcaXX64HAFy/fg2///4LVq36Emq1GkuXLsIvv+wx2awaYOgSEZEJRETU\nRVTUeaSnp8PW1hYREXVx/vw5nDx5HJMmTTMYGxNzHVeuXMakSa9BrVYhO1sDHx9fAEBYWDjmzn0T\n7dp1wBNPdDDqe7dt2w4AEBoahpCQUHh6egEAqlevgXv37sLNzQ3ffbcBBw7sAwDcu3cPN2/GoH79\nBgYrIv3779+4eDEKI0cOgSRJyM7OhpeXV0X/aAwwdImIFCZt7rslzkorg42NDfz9A/Djj7vQsGFj\nhIWF4/jxf3H79i0EBQUXGi0hJCQUn3zyZZFrL7///gqcOPEfDh7cjzVrvsSaNd+W+r1tbe0A6BZc\nsLW11b8uCAK0Wi2OHz+G//77F5999jXs7OwwbtwoZGdnF7MlCd26dceoUa+V40/AODyRioiITKJx\n46bYuHEdmjRphkaNmmDHjq0ID69TZFxgYDAePEjEmTOnAeh2N1+9egUAcPduLJo2bY4xY8YhLS0N\nGRnpcHJyQlpaWrnrSktLhaurK+zs7HD9+jWcPXtG/56trS20Wi0AoHnzR7F37+948OABACA5ORmx\nsbHl/r7F4UyXiIhMonHjpli79is0aNAQ9vYOsLe3L3I8F9DNit99dxGWL38fy5cvQnZ2Dp5//gXU\nqhWId96ZnRuwEvr3HwhnZxe0adMOs2a9gb/+2o+JE183OLGppJOc8t5r1ao1duzYisGDn0dgYBAa\nNGioH9OzZ2+89NJARETUxezZ8/Dyy2MwefJrEEUJtra2mDx5Gvz9/U32Z8Sl/R5CSctNWXsfSugB\nYB+WRAk9AMroQwk9AFzaj4iIyOIwdImIiMyEoUtERGQmDF0iIiIzYegSERGZCUOXiIjITBi6RERk\ndt99txFZWVlyl2F2DF0iIjK7zZs3Iisrs9j3RFE0czXmw9AlIqIK27BhDbZu1V0n+YMPlmLCBN2S\neseO/YN582YbjN2yZRPi4+MwbtxovPTSSwCAzp3bYeXK5Rg2bBDOnDmF/v17Ijk5CQBw4cJ5jBs3\nCgCQmZmJhQvfwciRL2H48ME4eHC/uVo0CV4GkohIgbyaNyj29YRjZ4p9vazjCyvL0n79+g3Et99u\nxIcfforQ0BqIi0tBZmYGGjRoiLFjJ+aOMry8Y94lHb/5ZjWaN38UM2a8hdTUVIwcOQQtWz4Ke3sH\no+qUG0OXiIgqrCxL++lIuTcdtVqN9u07Fnq/qH/+OYpDhw5g48Y1AHSLJdy9G4vAwGCT9VKZGLpE\nRApk7Ay1vOMLK9vSfkXZ2dkbLF6gVqshirrgzc7OP+FKkiS8++5i1KoVWKF65cJjukREZBLGLu0H\nAE5OzgbL9RVeeycgoDqios4DAPbt+0P/+qOPPoYtWzbpn0dHR5myhUpn1Ey3Y8eOcHFxgUqlgo2N\nDbZs2VLZdRERkZUxdmk/AOjZsxemTh2PgAB/LFmyssgSfUOHjsR7770DFxcXNG3avMDrL+ODD5bi\npZcGAgD8/QOwaNH/Kq8pEzNqab9OnTph27ZtcHd3N2qjFy9ehKdnQIWLk5OSlpuy9j6U0APAPiyJ\nEnoAlNGHEnoATLy0nyRJZfrc1IABA5CTk2P0eCIioqrAqNAVBAEjRoxA37598d1335U6/sSJE/jw\nQ+uZ7hMREZmDUcd0N23aBF9fXyQkJGDYsGEICQlBixYtHjq+Ro0aWLp0Ebp164769R8xWbFERETW\nzKhjugWtXLkSzs7OGDZs2EPH/PDDD3j22WfRvHlzHDlyBDY2/GQSERFRqWmYkZEBURTh7OyM9PR0\nHDx4EGPHji3xa7p3747nn38B3323EXPnvosJE6aYrGBzUdLBfWvvQwk9AOzDkiihB0AZfSihB8D4\nE6lKDd34+HiMHTsWgiBAq9WiR48eaNu2bakbfvfd97Bv3594//2F6NatOyIi6hpVEBERkVKVeiJV\nrVq1sHPnTuzYsQO7d+/GK6+8YtSGPTw88f77y5GdnY0JE8ZAo9FUuFgiIrJMsbF3MGTIAJNuMzr6\nIg4f/kv//ODB/Vi//huTbFuupQUr9YpU3bo9g759n8d//x3DqlUfVea3IiIimRW+wEVFXbp0EUeO\n5Idu27btEBn5kkm2XdLSgpWp0s9wmj9/Efbv34tFi95F165PP/SSYEREZN00Gg3eeWc2Ll68gNq1\nQzFr1tuwt7c3GHPr1k0sW7YYSUmJcHBwwHvvLYCLiw/++OM3fP3151Cr1XB2dsHy5R/jiy9WITs7\nG6dPn8TgwcOQlZWJCxfOYdKkaViw4G3Y2dkjOjoKiYkPMGPGW9iz53ucPXsa9es3wMyZcwAAS5a8\nh6ioc8jKykKHDp0wfPgrBksLenh4YMWKT/D330fw5ZefIScnBzVq1MTMmXPg4GD6lYsqPXS9vLyx\nePH/MGxYJCZMeBW7d/8MtVpd2d+WiKjKmjvXHrt3m/af9x49NJg7t+TdsTEx1zFjxhw0aNAQCxe+\ng+3bN2PgwMEGYxYvXoBp02aiRo2aOHfuDObOnYslS1bim2++wLJlH8HHxwdpaamwsbHByy+PRlTU\neUyc+DoAYM+e7w1m06mpKfj0069w8OA+vPHGJKxa9RVq1w7BiBEv4tKlaISFhWPUqNfg6uoKURQx\nYcIYXLlyyWBpQTc3NyQlJWLNmi+xYsXHsLd3wPr132DTpnUYOvRlk/4ZAmZaZah79x7o1asPduzY\nhs8//wSjR5d89jMREVkfPz9/NGjQEADQtesz2LLlW4PQzcjIwJkzJzF79hsFFjjQ3Tds2Bjz589B\nx46d0b79k0Z9vzZtngAAhISEwcvLG7VrhwAAatcOQWzsbYSFheP333/Grl07oNVqkZBwH1evXkVI\nSBgKLi149uwZXLt2BWPGjIAkSdBoNGjQoFHF/0CKYbYP0C5YsAQHD+7HggXvoEuXbrlNExGRqc2d\nm1XqrLQyFD6mW/gQrySJcHV1w5dfrte/lveRoalTZ+D8+bM4dOggRox4EatXryv1+9nZ2QEAVCqV\n/nHec61Wizt3bmPTpvVYvXotnJ1dsGDB2wbLBObXJaFly8cwZ867ZWm3XMy2tJ+Pjw/ee28pMjMz\nMWHCa2W6ljMREVm+2Ng7OHtWty7vr7/+jEaNmhi87+TkjICA6vjzz9/0r124cAGA7lhvvXqPYMSI\nUfDw8MS9e3fh5ORksPxfSYq7zlNaWhocHR3h5OSMhIT7OHLkkEEtedt+5JGGOH36JG7dugkAyMrK\nxI0bMWXo3HhmvVRUz5690aPHduzevQOrV3+KkSPHmPPbExFRJQoKCsa2bd9h4cK3ERwcgl69+hUZ\nM2fOu3j//YX45psvodVq0LNnD/Tv/yI+/ngFbt68AQBo3rwlwsLCUa2aH9at+xrDh0di8OCHXwUR\nKP7M6bCwcISHRyAysh+qVfNDo0aN9e/lLS3o4+OLFSs+wcyZczB37kxkZ+dAEASMHDkGtWoFVvBP\npJg6y3oZSGM97AojcXFxeOKJlsjMzMSffx7S74O3NEq6Soq196GEHgD2YUmU0AOgjD6U0ANg4qX9\nTMnX1xcLFy5Beno6Jk0ay93MRERUZZg9dAGgV6++ePrpZ3Ho0EF8/fVqOUogIiIyO1lCVxAELF78\nP3h4eOCdd97C9evX5CiDiIjIrGQJXQDw8/PD/PmLkZ6ehsmTxxV75hkREZGSyBa6ANCv3wB06dIN\nBw7sw5o1X8lZChERUaWTNXQFQcCSJSvg7u6BuXNnVdrnooiIiCyBrKELAP7+AZg3byHS0lK5m5mI\nyEoZu7Tfnj3f4/79eDNUZJlkD10AGDBgEDp16ox9+/7Ehg1r5S6HiIjKwZil/X78cTfi4uKKfa8q\nfITUIkJXEAQsXfoBXF3d8NZbM3H79i25SyIiojLKW9pv8OD+mD17epFF4vfu/R0XLpzHvHmzMXx4\nJLKystCxY0d88smHGDHiRfz5528YN24UoqJ0l4ZMSkpE//49AegC+eOPV2DkyJcwdOgg7Nq13ez9\nmYJFhC4AVK9eA++8swApKcmYMmU8dzMTEVVA8+bOxd5MNb44MTHX0afP81i3bjOcnJywfftmg/c7\ndOiEevXqY86cd/Hll+v1a+26u3tg9eq16NSpSzFb1c2ev/9+J1xcXPH559/g88+/wa5d2xEbe6dM\n9VkCiwldABg06EV06NARv//+K779doPc5RARURkUXtrv1KmTRcZIkoTCc6pOnTqXuu2//z6Cn376\nAcOGDcIrr7yE5OQkqzz51qwLHpRGEAQsW/Yh2rV7DLNnz0CHDh3h7x8gd1lERFbn2DHjVucp7/ji\nlLa038M4OjrqH6vVakiS7thudnZ2gVESJk16HS1bPlbRMmVlUTNdAKhZsxbmzJmHpKRETJ06gbuZ\niYisRGlL+wGAs7Mz0tJSH7qNgIAauHDhHAAYLAH46KOPY9u2LdBoNACAGzdikJWVacryzcLiQhcA\nhgwZhieeaI9ffvkJW7Z8K3c5RERkhLyl/QYP7o+UlORil/Z7+ulnsWTJQv2JVIVnxy+8EInt27di\n+PDBSE5O1r/eo0cvBAfXxogRgzFkyAAsWbIQWq220nsyNbMv7WesmJjraNfuMdjZ2eLAgX/g5+dn\nosqMo6Tlpqy9DyX0ALAPS6KEHgBl9KGEHgALXtrPWIGBQZg9+20kJiZi2rRJ3M1MRERWz2JDFwCG\nDXsZrVu3xZ4932PHjq1yl0NERFQhFh26KpUK//vfSjg5OWHGjKm4d++e3CURERGVm0WHLgDUrh2C\nN9+cg4SEBMyYMVXucoiIiMrN4kMXAEaMGIVWrR7H7t07rPbSX0RERFYRuiqVCitWfAQHBwdMnz4F\n8fFVd4UKIiKyXlYRugAQEhKGGTPeQnx8PGbO5G5mIiKyPlYTugDwyitj0KLFo9ixYxt++GG33OUQ\nERGViVWFrlqtxooVH8Pe3h7Tpk1CQsJ9uUsiIiIymlWFLgCEh9fBG2/MQlzcPbz55htyl0NERGQ0\nqwtdABgzZiyaNWuOrVu/w08//Sh3OUREREaxytDV7Wb+BHZ2dnj99YlITHwgd0lERESlssrQBYCI\niLp4/fUZuHs3FrNnz5C7HCIiolJZbegCwGuvTUDjxk3x7bcb8OuvP8ldDhERUYmsOnRtbGzwwQef\nwNbWFlOnTkRSUqLcJRERET2UVYcuANSrVx+TJ0/DnTu3MWfOm3KXQ0RE9FBWH7oAMH78ZDRo0Agb\nNqzFH3/8Jnc5RERExVJE6Nra2uKDDz6BjY0NJk8eh5SUZLlLIiIiKkIRoQsADRo0xMSJU3H79i3M\nnTtb7nKIiIiKUEzoAsDEiVNRv34DrF37Ffbt+1PucoiIiAwYHbqiKKJ3794YPXp0ZdZTIXZ2dvjg\ng4+hVqsxefI4pKamyF0SERGRntGhu2bNGoSGhlZmLSbRqFETjB8/CTduxGDevDlyl0NERKRnVOjG\nxsZi37596N+/f2XXYxKTJ7+BunXr4auvvsDBg/vlLoeIiAiAkaG7YMECTJs2DYIgVHY9JmFvb48V\nKz6GSqXCxIljkZaWJndJREREsCltwN69e+Hj44N69erh6NGjRm/Y19e1QoVVVJcuHTBt2jS89957\nWLZsAT744IMyb0PuHkxFCX0ooQeAfVgSJfQAKKMPJfRgLEGSJKmkAcuWLcOuXbugVquRlZWFtLQ0\ndO7cGYsXLy5xw3Fx8p/ElJmZiaeeegIXL0Zh5849ePzxNkZ/ra+vq0X0UFFK6EMJPQDsw5IooQdA\nGX0ooQfA+P84lLp7efLkydi7dy9+//13LFu2DK1atSo1cC2Fg4MDli//CCqVChMmvIr09HS5SyIi\noipMUZ/TLU6LFo9i9OixuHbtKhYunCd3OUREVIWVKXQfffRRrFq1qrJqqTRvvPEmQkPD8NlnH+Po\n0SNyl0NERFWU4me6AODo6Ijlyz8GAEyc+CoyMjJkroiIiKqiKhG6ANCq1WN45ZUxuHz5EhYtmi93\nOUREVAVVmdAFgBkz3kJwcG2sWrUS//77t9zlEBFRFVOlQtfJyQkrVnwMURQxYcKryMzMlLskIiKq\nQqpU6ALA44+3wcsvj0J09EUsWfKe3OUQEVEVUuVCFwDefHMuAgODsXLlchw/fkzucoiIqIqokqHr\n7OyM5ctX6nczZ2VlyV0SERFVAVUydAGgbdt2GDp0BC5cOI///c86rrBFRETWrcqGLgC89dY7qFUr\nECtWLMOpUyfkLoeIiBSuSoeui4srli37EFqtFuPHv4rs7Gy5SyIiIgWr0qELAO3bP4kXXxyGc+fO\nYPnyJXKXQ0REClblQxcA5s6dhxo1amL58iU4c+a03OUQEZFCMXQBuLq6YenSD6DRaDB+/Bjk5OTI\nXRIRESkQQzdXx45PYdCgF3HmzCl8+OH/5C6HiIgUiKFbwNtvz4e/fwCWLl2E06e5m5mIiEyLoVuA\nu7sHli5dgZycHAwdOhSpqalyl0RERArC0C2kc+duiIwcgv/++w8DBvRGcnKS3CUREZFCMHSL8f77\nyzFo0CD8889R9O3bEwkJ9+UuiYiIFIChWwwbGxusWbMGgwa9iJMnj6N372cRFxcnd1lERGTlGLoP\noVarsWzZhxg+fCTOnz+LXr2exp07t+Uui4iIrBhDtwQqlQoLFy7Bq6+OR3T0RfTs2Q03bsTIXRYR\nEVkphm4pBEHAnDnzMGXKG7h+/Rp69uyGK1cuy10WERFZIYauEQRBwBtvvIlZs+bi1q2beO65pxEV\ndUHusoiIyMowdMtg/PjJmD9/Ee7ejUWvXk/j9OlTcpdERERWhKFbRiNHjsGSJSuQkJCAPn2exfHj\nx+QuiYiIrARDtxyGDBmGDz9chZSUZPTt2xNHjhyWuyQiIrICDN1yev75F/DZZ18hMzMDAwf2xoED\n++QuiYiILBxDtwJ69uyNr75aD41Gg0GD+uG3336WuyQiIrJgDN0K6tr1aaxb9x1UKhVeemkQfvhh\nt9wlERGRhWLomkCHDh2xceNW2NnZ4+WXh2Dbts1yl0RERBaIoWsirVu3xebNO+Ds7IIxY17Ghg1r\n5S6JiIgsDEPXhFq0eBTbtu2Gp6cnJk58DatXfyZ3SUREZEEYuibWqFETbN/+I3x9q2HGjKn4+OMP\n5S6JiIgsBEO3EtSrVx87d+5BQEB1zJ37JpYuXQRJkuQui4iIZMbQrSRhYeHYuXMPAgODsGjRfCxY\n8A6Dl4ioimPoVqLg4NrYuXMPQkJCsWLFUsyePZ3BS0RUhTF0K1mNGjWxc+dPqFu3Hj777BNMnToR\noijKXRYREcmAoWsGfn5+2L79RzRo0Ahr136FceNGQ6PRyF0WERGZGUPXTLy9vbFt2240b94Cmzdv\nwujRI5CTkyN3WUREZEYMXTPy8PDE5s078fjjbbBr13YMHz4YmZmZcpdFRERmwtA1MxcXV2zcuBXt\n2z+Jn3/egyFDBiI9PV3usoiIyAwYujJwcnLC2rXfokuXbti79w8MGtQPqakpcpdFRESVrNTQzc7O\nRv/+/dGrVy/06NEDK1euNEddiufg4IAvv1yHHj164dChg+jfvxeSkhLlLouIiCqRTWkD7OzssGbN\nGjg6OkKr1eKFF15Au3bt0KhRI3PUp2h2dnb49NMvYW9vjy1bvkWfPj3w3Xc74O3tLXdpRERUCYza\nvezo6AhAN+vlR11My8bGBitXfooXXxyK06dPok+f7rh7967cZRERUSUodaYLAKIook+fPoiJiUFk\nZGTps9zgYHiJRa+8lHDsTLHDvZo3KPZ1WcerhCI9VGY9XwFwGDkan3++Cr16PY2tW3ejevUaFd9+\ngT6s6s+/oNweLKaeco5HzHWLqofjOd4SxisiL4CH/v0uzKjQValU2LFjB1JTU/Hqq6/i0qVLCAsL\nK/Fr1CqhyGu+vq4P+QZFx1rC+MI9VHY9n376Mby83LFo0SL07v0M/vjjDwQHB1d4+3l9yP3nWZHx\napVgUfWUZ/xDv8ZK6i843uBrLaCe8ozXP7eQeso7vrh/a+Wsp8zjoYy8MJYglfFiwCtXroSzszOG\nDRtW4ri4OOs+G9fX11WWHiRJwtKli7B48QJUr14D27btRkhIyf/BKYlcfZiSEnoA2IclUUIPgDL6\nsPgeRBHIzISQmQEh9x4ZmRCyMiFkZgKZGRAyMuE+dJBRmyt1ppuQkABbW1u4uroiMzMThw8fxiuv\nvFLhPqh4giBg6tTpcHBwxDvvzEbPnk9jy5ZdqFu3ntylERHJy8gAzHtfN7bg89z3szLzt5M7Hpm5\n28nIHZf3fna2cbWZKnTj4uIwffp0iKIIURTxzDPPoH379sYVQeU2duwEODo6YMaM19G79zP47rsd\naNiwsdxlEREVJUlAdjaE9DQI6em5tzT9PdLTIaQ95D1JA9fEFMMAzMrSPy9XAJa1fEEAHB0hOThA\ncnCE5OICyccXkoM9JAdHIO91BwdIjo6Avb3hcwcHuBj5vUoN3YiICGzfvr2CLVF5jBgxCg4Ojpg8\neRz69OmBTZu2onnzlnKXRUTWSJKAjIwioWd4nw4UfC2taEgWHZf7ulZb7tIcCpZZXAB6+0BydCga\ngA4ORQPRwQGSfe57jo4FxjoCDvaGz/O2aWsLCGU7NluYyUKX5BUZOQQODg4YO3YU+vV7Dhs2bMbj\nj7eRuywiqkyiqAuylBQIqakQUpINH6emQJWaCkg5cI5/UCQQ9Y/T8h8jIx2CCdbzllQqSE7OkJyc\nACcniN4+kJyc9K9JTk6QnAs8dnIGDN43fM+rpi/i00VdANo7AHZ2FQ5AS8bQtQJ9+z4POzt7jB49\nHAMH9sGaNZvQvv2TcpdFRAVJku64YEpKbiimFB+aqbrHKv17KbrX8h6npEBISzU6IJ2KK8XWVh9u\nors7pIDqucH38PArGJYlhSTs7U0bir6ukCz5RCoTY+haiR49noODw3oMH/4iBg9+HqtXr0GXLk/L\nXRaR9cvJyZ095oeeKi0lPwALhmZaam5gFgzRlPyvL+fFgyQ7O0iurpBcXCEGBUN0dc197gLJxS3/\nsasrJFfc0+i4AAAgAElEQVQ3iC4ukFxc4FHTDwlZAJxzw9HRUReMtram/TMik2HoWpHOnbth/frN\nGDJkIIYOjcSnn36JHj16yV0WkbxEEUJyEoTERKiSEiEkJkJISoQqMbH415ISgdRkeCcl6YKynMtr\nSmo1JBddOIoB1SE560JRdHXLD0gXV/2Y/OB0g+iS/1hycdHNHsvD1xXaKjRLVAKGrpVp164DNm3a\nhkGD+mPkyKH48MNV6N9/oNxlEVWMkcGpSnxQJECF5KQyHauUnJwAd3eInl6QagUWmUnqQzMvLAuG\npqsrRGfdPRwdFX3skSoHQ9cKPfZYa2zZshMDBvTB2LGjkJWVhcGDX5K7LKrqSgzOB/qQzAvSigan\n6O4BsXp1iPXqQ/LwgOTuAbHQveThAdHDE5KHJ0R3D0ju7oC9PXx9XfGAM0SSAUPXSjVr1gLbtn2P\n559/DpMnj0NmZgZefnm03GWRUmRkQBUfB9X9eKjux0OIj4fq/n2o7scDmalwi40zTXB6eEKsXgNi\n/UfyQ1Iflh6FXvPUvwc7u0psnqjyMHStWMOGjbBjxx707dsDM2dOQ0ZGJsaNmyh3WWSJ0tL0AaoP\n0fgCz+/H54bsfaji43UXLShB3hFIyckZoocHg5PISAxdKxcRURe7du1B3749MW/eW8jISMfrr8+A\nwGNNyiVJhiEaHwchNyzzn+cFqm52KqSnl75Ze3uI3j7QhIVD8vaG6O2ju/n4QPLxzX3uDc/QWojX\n2up21TI4icqEoasAISFh2LlTN+NdsuQ9ZGZmYvbstxm81kKSdB9FiYszDMr4eMNdvPfv658bc8at\n5OCgC9HwiEIh6gvJx0cfonnPJWcX404MqmKfqyQyJYauQgQGBmHXrp/Qt28PrFy5HBkZ6Zg/f7Hc\nZVVdkqQ7eSg2Fqo7t6G6GwukJcL5+q1Cx0lzH2dllb5JR0eIPr7Q1K2nuwpQboDqZ6O5AZoXrnB2\n5tm1RBaGoasgAQHVsWPHHvTv/xxWr/4MWVlZ+Prr1XKXpTzp6VDF3oH6bm6g6oP1DtR37kAVeweq\nu7HFzkYLXj1IcnKG6OMDTf1HdCFaIDAfGqJEZNUYugpTrVo1bN/+PQYM6IN1677BvXt3MG/eYtSu\nHSJ3aZZPo4Hq3l1daBYIT/Wd27rHsXd0AZuU+NBNSCoVRN9qutmof4D+pg2oDrewIDywdc4PUafi\nLuBHRErG0FUgLy9vbN26C6+8Mgy//PIL9u/fjwkTpmDs2ImwL++Vb6yZJEF4kKAL0oKz0dhYqGIL\nzFTj7pX4kRfRwwNiQAA0TZvlBmkARL8AiAHVIfr76+59fAGbh/y18nWFhsdCiao0hq5Cubm5Y+PG\nrfjzzz2YMGEiFi2ajy1bvsWiRcvQrl0HucsznbQ0qO8WmJnmBqsqNm+GGgvV3TslHjOVHBwg+gcg\np9XjEIsJUq2fP0T/AN0ViIiIKoChq2CCIGDAgAFo0aINFi2aj9WrP0O/fj3Rp08/vP32Qvj5+cld\n4sPlnoikvhEDJMXB4eKV/BlqgWBVJSc9fBMqFUQ/f90xU78AXaDm7uoV/fz1wSq5e/CEIyIyC4Zu\nFeDm5o758xdjwIBBmDZtErZt24Jff/0FM2fOxtChL0OtVstTWFoa1DdioI65BlXMdaivX4c6RndT\nxVyHKiVZP9S10JeKnp4Qa9SEpnkLaP0Dip2hij6+gFy9EREVg6FbhTRq1AQ//PAb1q79GvPnv40Z\nM17Hpk0b8P77/0OTJs1M/w2zs6G6dVMfpLowvaZ7fP06VPFxxX6Z5OQEbWAQcgJbQxsYBKd6dZDs\n6gWtf26g+gcADg6mr5eIqJIxdKsYtVqNoUNH4JlneuDtt2dh8+ZN6Nr1SQwdOgIzZ74Fd3cP4zcm\nirqPzsRch+r6NYNZqjrmOlR3bkMQxSJfJtnaQluzFjSPNIA2MAjawCCIuffawGBIPj4Gu3udfF2R\nxROQiEgBGLpVVLVq1fDRR59h0KAXMW3aJHz11Rf4/vtdePvt+ejb93nd1awkCUJCAtS5s1OVfvdv\n7u7gmzcgZGcX2bYkCBADqiPn0ccKhGkQxKBg3b1/AHf7ElGVxNCt4to2boIDH32OXz/7GCd3bkPW\nqyNxcdZ0NPX0hGNsLFRpqcV+nejjkztTDS4UrEHQ1qhV/kW5iYgUjKGrdFlZUF+OLjBLzdv9mzt7\nTUgAAAzOvQEAEu4jOeE+Yn184dGmLVA7JDdYdTNVba1AwMVFro6IiKwWQ1cJJAlCfDxsoqOgvhgF\ndXQUbC5GQX0pGrh9C17FXPBBsreHtlYgNI2b5odpkC5Qf4m+iCnz38btO7cReOEC3hs6Ak891VWG\nxoiIlIWha01EEapbN2Fz8QLUFy/mh2t0FFQPHhQZrq1eA2jfHhkBNQ1OVBKDgiBW8wNUqmK/Taem\nzXHwmR5YunQRPv30Iwwa1B/du/fEu+++hxo1alZ2l0REisXQtUQ5OVBfvQL1xagCs9eLsLl0sci6\nqJJKBW1wbeS0ehza8Aho6kRAWycC2vA6kFxc4evritRynPnr4uKCOXPm4fnnX8C0aZPwww+78Oef\nv2PatJkYOXI0bG1tTdUtEVGVwdCVU1oabC5dzA/V3Fmr+uoVCBqNwVDJwQHa0HBo6tTJD9fwCGhD\nQiv1pKV69epj5849+PbbDXj77VmYO/dNfPvtBixe/D+0avVYpX1fIiIlYuiagXD/ftHjrdEXob55\no8hY0d0DmibN8kO1Th1owiMg1gqU7WM2KpUKL7wwGF27Po13352Ldeu+QY8eXRAZOQSzZ78NLy9v\nWeoiIrI2DF1TkSSobt8qsEv4ItQXL8AmOgqq+/eLDNf6+SP7iQ76UNXWiYAmPAJStWoWex1gLy9v\nLFv2IQYMiMS0aZOwfv0a7NnzPd56ax4GDoyE6iHHiImISIehW1YaDdTXrhaatUZBHR1d5DOtkkoF\nMTAIWc1bFtglXAfaOhGQ3NxlaqDiWrV6DL/9th9ffPEpFi2aj4kTX8OGDWuxePH/UL/+I3KXR0Rk\nsRi6D5OeDpvTJwuEq+5sYfWVyxBycgyGSnZ20IaGI7tAqGrCI6ANDVPsNYJtbW0xZsxYPPdcb8ya\nNR3ff78TnTq1xahRr2Hq1Olw4ed4iYiKYOgCEFKSYXPqJGxOnoDNqeOwOXkCuHIZnoU+3yq6uELT\nsBG0deoW2CVcB2JQcJW9rGH16jXw5Zdr8dtvP2P69Nfx8ccfYMeOrZg/fzGeeeZZ3eUkiYgIQBUM\nXSE5CTanT+kC9uR/uvsrlw3GiG7uQLt2yKgdVuCEpgjdNYMZIsV66qmuOHCgHVasWIIPP1yOYcMi\n0blzVyxY8D6CgoLlLo+IyCIoOnSF5KQiM9giAevugewn2kPTqAk0TZoip1ETiMG14VvNrVyfb63K\nHB0dMX36bPTtOwBvvDEZv/76Mw4e3I9Jk17Hq6+Oh52dndwlEhHJSjGhW6aAbdwUmsZN9AHL2atp\nhYfXwdatu7F163eYM+dNLFjwDjZv3oRFi5ahbdt2cpdHRCQbqwzdIgF74jhsrl4xGKML2A7QNG7C\ngJWBIAjo128AOnfuioUL5+Grr75Anz7Pol+/AZg7dz6qVasmd4lERGZn8aGrD9gTx/NnsKUFbOOm\nupObGLCyc3f3wHvvLcWAAYMwbdpkbNnyLX755Se8+eYcDBkyDOoqegIaEVVNFhW6QlJi0V3EJQRs\nTpOm0DRqwoC1Ak2bNsdPP/2Br79ejQUL3sEbb0zGpk3r8P77y9GoURO5yyMiMgvZQteogPXwQHa7\nJ3Nnr00YsFZOrVZjxIhX8OyzPTFnzkxs27YFXbp0wPDhIzF9+iy4WfEFQ4iIjGGW0DUI2JPHYXvy\nONTXrhqMYcBWHX5+/li16ku88MKLmD59Cr744lPs2rUD8+YtRK9effnZXiJSrMoJ3T/+gOPev2Bz\n6oRxAdu4KcTAIAZsFdO+/ZPYu/cwVq5cjuXLl2DUqOFYv34tFi1agtDQcLnLIyIyucoJ3U6dkHcR\nQIOAzTsGy4ClXPb29pgy5Q306dMfM2ZMxR9//Ib27R/HuHGTMGHCFDgo9DKaRFQ1VU7oTp+OpPD6\nDFgyWu3aIdi4cSu+/34XZs16A0uXLsLWrd/lnvncW+7yiIhMotS12GJjYzFkyBA888wz6NGjB9as\nWVP6VhcuRHaPXjwmS2UiCAJ69HgOf/31D0aNeg03bsRg4MA+6NevH44d+wdSoWthExFZm1JDV61W\nY8aMGfjxxx+xadMmrF+/HpcvXy7ty4jKzcXFFfPmLcSvv+5HixaPYuvWrXj66U7o0OFxfPbZx0hI\nKLo+MRGRNSg1dH19fVGvXj0AgLOzM0JDQ3Hv3r1KL4yoQYOG+P77X/DTTz+hZ8/euHQpGrNmTUej\nRhEYNWoY9u/fC1EU5S6TiMhoZTqme/PmTVy4cAGNGjWqrHqIDKhUKnTt2hXNmrVGfHw8Nm/ehHXr\nvsb27VuxfftWBAUFIzJyCAYOjIS/f4Dc5RIRlUiQjDxQlpaWhhdffBGvvvoqnnrqqRLHBgej2BnI\nsWNpxY5v3ty52NflHK9SqYr0YE315ynYhyXUU57xeT3kjZckCX//fRTr13+DXbu2Iz39LADAwcER\nLi4ucHBwgCAIFlN/npgYFeKKWbnK0v/8C4/39XU16EPuesozvmAPllBPecf7+roiMLD4vT3WUD8A\ntGzpavV5Aej+fhvDqJmuRqPB+PHj8dxzz5UauHlUqqIF+Pq6PmRs8duQe3zhHuSup7zj8/qwlHrK\nM16lUhmMf/bZznj22c5ISkpCSIgKqakpyMzMQGZmBtRqNVxcXJCUdB9hYWEWUX9JX2MNf/6Fxxd8\nbAn1lGd83nNLqaf844v/AmupX/c11p8XxjJqpjtt2jR4enpixowZRm+4uP/RW5PC/5u3Vkrow9ge\nTp8+hQ0b1mDLlu+QlJQIAGjbth0iI4ege/eesn/mVwk/C0AZfSihB0AZfSihB6Dk/1QUVGpGHzt2\nDLt378aRI0fQq1cv9O7dG/v3769wgUSm1rBhIyxcuASnTkXh448/R5s2T+Dgwf0YM+ZlNGpUBzNn\nvo6zZ8/IXSYRVWFGH9MtK2v/n4uS/vdl7X1UpIcrVy5hw4Z12LhxHeLidGfdN2vWHJGRL6F3775w\ncTHuf6emoISfBaCMPpTQA6CMPpTQA2DCmS6RNQsJCcOsWXNx4sR5fPPNRnTp0g0nThzHlCnj0aBB\nHUyc+Br++ecoL7xBRGbB0KUqwdbWFk8/3R3r1n2H48fPYcaM2fDx8cWGDWvRvXtntGvXCqtWrcT9\n+7zwBhFVHoYuVTkBAdUxadLr+PvvE9i8eSd69eqDq1ev4K23ZqJRozoYOXIo9u79gxfeICKTk20R\neyK5qVQqtG//JNq3fxL379/Hli2bsH79GuzcuQ07d25DYGAQXnhhMF54YTCqV68hd7lEVMmys4H0\ndCA9XShwr3uclpb/WkZG0TEbNxr3PXgi1UMo6eC+tfdhzh4kScKxY/9g/fo12L59K9LT06BSqdCx\n41OIjHwJXbp0g62tbbm2rYSfBaCMPpTQA6CMPsrSgyiimMDLv8/IKDksC79XOEA1mvIv0GNsknKm\nS1SAIAho0eJRtGjxKObNW4gdO7Zh/fpv8Ntvv+C3336Br281DBgwCIMHD0FISNELbxCRjiQBaWlA\naqqAlBQBKSmGj9PSdI+1WiA+3v6hQVg4LE1BpZLg5AQ4OenuvbzEAs91rzk7S3B0zB9jeF/0NehX\nkS8ZZ7oPoYT/QQLK6MMSejh37iw2bFiDzZs34cGDBwCA1q3bIjJyCJ599jk4OjqWug1L6MMUlNCH\nEnoATN+HJAFZWXnhmB+SqanIDUvd4/zwzH+vuK+RpPKHpL19ySFX8N7R0XCMs/PDw9LREbC3N/2q\ns8Z+ZIih+xD8S2k5LKmHzMxM7NnzPdatW4MDB/YCANzc3NGv3/OIjHwJDRs+fDEQS+qjIpTQhxJ6\nAPL70GhQKAx1j4ubZRYOybzHea/n5JQvjezsJLi6SnBxAVxcdI9dXQFXVwnOzvmPdWN0z11cJNSs\n6YTs7LQiM0y12sR/WJWMoVtBSvtLac0stYdr165i48a12LhxPWJj7wAAGjduisjIIejTpx/c3NwN\nxltqH2WlhD4srQdJ0h2rTEwUkJgoICkp7x548CD/eeH309JUSE6Wyr3bVRAMw9DZuWAwokBA5j/P\nC1NdkOaHp719+Xq3tJ9FeTF0K0hJvwjW3oel96DRaPD7779i/fpv8OuvP0Or1cLR0RE9e/ZGZORL\naNXqMQiCYPF9GEsJfVRWD5mZKBSQMAjJwqFZ8P3sbOOD09ZWgru7BC8vFRwdtUVmjwXDMO/1ggGa\n956Tk+l3s5aVEn6fAIZuhSnpF8Ha+7CmHmJj7+Dbbzdg/fo1uHbtKgAgLCwckZEvYeTIobCzc5O5\nwoqzpp/Hw5TUQ04ODELz4YFZ9P3MTOMTTK2W4OEhwd0d8PCQ9Dd3d6nQ86Lv54Wl0n8W1oShW0FK\n+kWw9j6ssQdRFHHo0EGsW/cNfvhhF7KysgAAoaFhaN26rf4WEFBd5krLzlp+Hnlnz8bHC7h/X0B8\nvID4eBXu3xeQnm6PO3dy9KFZcBduWXbVCoIuFN3dJXh65gem4fOi73t46HbXVnSWaS0/i5IooQeA\noVthSvpFsPY+rL2HBw8SsG3bZhw48Cf27z+A1NT8XkJCQvUB3KbNE1YRwnL+PDIzDUM0Li7vsapQ\nuOoeZ2QYl2pubg+bZepC82GzUFfXsq+nakrW/ncDUEYPAEO3wpT0i2DtfSihB0DXx507D3DmzCn8\n9ddBHDp0AEeOHEZKSrJ+TO3aIWjT5gk8/ngbtGnzhEVeCcuUP4+cHCAhQReexYWmLlhV+sepqaWH\nqL29BB8fw5u3twQfH1H/PDTUCZKUCg8PCW5ugI2VXrFACX83lNADwNCtMCX9Ilh7H0roASi+D61W\naxDChw8fMgjh4ODaBiFco0ZNc5ddREk/D61Wd7Zt4QDNn5EWfE+FxMTSQ9TGJi808wPU17domOa9\n7uxc+m5bJf9OWRsl9AAwdCtMSb8I1t6HEnoAjOtDq9Xi7NnTBiGcnJykfz8oKNgghGvWrFXZZUOj\nAe7dE3DnjoDYWBUyMx1x7VpWkRlpfLyAhAQBolhy4qlUEry8Cs9Ciz7OC1N398q5kEFV+Z2ydEro\nAWDoVpiSfhGsvQ8l9ACUrw+tVotz587gr78O4NChgzh8+BCSkhL17wcGBqNNm/wTs2rVCizT9tPS\ngNhYAbdvq/SheueOgNu38x/fu1d6kHp46EKy5Bmp7ubpKcl+4YOq/DtlaZTQA8DQrTAl/SJYex9K\n6AEwTR+6ED6LQ4cO4K+/DuLw4b8KhXAQWrdui8cea4v69dtDrQ7MDVEVYmMF3LmjC1LdTYXk5IeH\nqZ2dBH9/CQEBIgICJAQESPD3FxEW5gBb23T4+OhC1ctLQjnXgJANf6cshxJ6AIwPXSs9fYCoalKr\n1ahTpxHc3BqjcePx6NVLwokT93DqVDyuXMnErVu22LTJD5s21QBg99DtuLtLqFFDRPPmulD195dQ\nvXr+44AA3ey0uN26vr4OiIvTVl6TRArG0CWyEJIEJCejwK5e3Wy04K7e2FjdCUiGaufedMdLfXyy\n4eAQj5yca0hMPI2srCsAbgG4CT8/EW3ahKB9+1Zo3botAgODIMh9SSKiKoShS2QGWi1w6xZw5oyq\nwK7egrt7da+VdGEGJyfdDLRuXU3uzFTM3eWrm6FWr67b3as7XuoKoCFE8RGcP38Ohw8fxF9/peDw\n4YPYtu0Atm37BgBQo0ZN/WeEW7dui6CgYIYwUSXiMd2HUNJxBmvvw1p6SEkBrl9X4do1Fa5fF3D9\nukr//ObNkldv8fExPG4aEKAL1bxdvQEBItzcKn4WryiKuHDhfG4IH8Thwwdx//59/fs1atTUnxnd\nunVbBAfXLhLC1vLzKIkSegCU0YcSegB4IlWFKekXwdr7sJQetFrdmb7Fher16wLu3y/+0kQ+PiKC\ngiSEhqrh5ZVtcGJSQIAIP7/yr9BSUaIoIirqAg4dOoBDh/7CoUMHDEK4evUaBiFcu3YIqlVzs4if\nR0VYyu9URSmhDyX0ADB0K0xJvwjW3oc5e0hNhT5M84JVF6oq3LhR/EowtrYSAgMlBAWJ+ltwcP5z\nFxfz91FekiQhKuoC/vrrAA4f1oVwfHy8/n1//wA0bdoEgYEhqFMnAuHhEQgPrwNvb28Zqy47a/hZ\nGEMJfSihB4BnLxMVSxR1s9W8UL12LT9Ur18v7iQlHW9vEQ0aFAxV3ew1KEg3a5X7c6emIggC6tat\nh7p162HEiFcgSRIuXozSh/CRI4ewZ8+eIl/n7e2tD+D8WwRq1qwFlZwXJyayMAxdUpz0dBjMVAvu\nAo6JUSErq+hs1cZGQq1aEho00BQJ1aAg3fHUqkgQBERE1EVERF0MHz4SAGBjo8GRI/8hOvoiLl6M\nwqVLuvu//z6CI0cOGXy9o6MjQkPDUadOnQKhHIGQkFDYy7VPnUhGDF2yOpIE3L1reGy14Gz13r3i\nZ1aenhLq1Ss6Uw0K0p35a60XvTc3T09PtGjxKFq0eNTg9czMTFy9egXR0VEFwvgiLl+OxpkzpwzG\nqlQqBAUFo06dCISF1cndVa2bIbu7e5izHSKz4j8zZJEkSbcb+MIFFWJjgTNn7PWhGhOjKnbJNrVa\nQs2aEtq10+hDVXevu7m7y9BIFeLg4IB69eqjXr36Bq+LooibN2/khvFF/cw4OjoKP/+8Bz//bLi7\nulo1v9wwDjc4bhwQUJ0fZyKrx9AlWUmS7mL6Fy6oEBWlu124oMbFiyokJRX8B1Z3dSU3Nwnh4WKB\nMJX0M9caNThbtUQqlQqBgUEIDAxCp05dDN67f/8+oqOj9Luqo6OjcOlSNA4e3I+DB/cbjHV2dkF4\neDjCwyMMZsjBwbVha23XoaQqi/9EkdnExeWHa37Iqoss76ZWSwgJEfHEEyIiIkS0bGkPb+80BAWJ\n8OCeR0Xx9vaGt3drPPZYa4PX09PTcflydIEw1s2Qz507ixMnjhuMtbGxQe3aIQXCOFx/7+Ji3Bml\nRObC0CWTu39fKBSsulvhz7GqVBJq15bQurUGdevqAjYiQkRoqGjwuVVfX3vExYlm7oLk5OTkhIYN\nG6Nhw8YGr2s0GsTEXEN0dLTBSVzR0RcRHX0RP/6422B89eo1DM6mzpsh+/i4mLMdIj2GLpXbgwdA\nVJS60K5hVZGP3QiChKAgCS1b5hiEa1iYCAcHmYonq2RjY4OQkDCEhISha9en9a9LkoR79+4VOYkr\nOjoK+/b9iX37/jTYjouLC/z9A+DvHwA/P//cez+D1/z8/OHk5GTuFknhGLpUqqQk4MIFtUGwRkWp\nij1LODBQRJcuGkREaBERIaJuXV248t8uqkyCIMDPzw9+fn5o27adwXupqSkFPt6kmyHfvHkdt2/f\nxqVL0SVu193dA/7+/vDzC4C/v39uKPvnhnL+Y378iYzF0CW9lBTkBqraIFxjY4uGa61aIp56SpM7\na9Wibl0R4eEinJ1lKJyoBC4urmjatDmaNm2ufy3vKkhZWVm4d+8u7t6NRWxsLO7evYPY2FjExt5B\nbOyd3NfvICrqQonfw8vLq5hgDjAI6WrV/HjCFzF0q6LUVODixfwzhfPC9fbtouFao4aIjh01ubNW\n3ey1Tp38SxsSWTN7e3vUqhWIWrUCSxyXkZGBe/fuFgjm/HDOC+Zbt27i/PmzD92GIAjw9vbRB3HB\nXdsFw9nHxxc2PA1fsfiTVbD0dODff4HDh230s9eoKBVu3CgargEBIjp00Oh3CeftHnblyZ9EcHR0\nRFBQMIKCgkscl5aWhrt3Y/VBnB/M+Y+vXLlc5GIhBalUKvj6Vis0Yy46g7a2612TDkNXITQa3a7h\n48fVOH5chf/+081gRREAHPXjqlUT8cQT+WcL581eeeEIoopzdnZGSEgoQkJCSxyXmppisBu74K7t\n/F3a53Hy5PGHbsPGxgZeXl5wc3OHu7s73N09Ctx76J97eHjAza3ovVopFwy3MgxdKyRJQEyMgOPH\n1fjvP13InjqlNrhKk5OThJYttWjZ0gaBgZn62aunp4yFExEA3XHmsDBXhIWFP3SMJElITk4q9hjz\n3bt3ERt7B8nJiUhIeICYmOvIzs4uUw2urm7FhLW7QVjnv+ZpEOCOjo68Olg5MXStQEICcOJEXsDq\nQrbgx3JUKgl164po1kyLZs1ENG2qm73a2OSdMJIjY/VEVB6CIOhnrBERdYsdk3dCmCRJyMjIQHJy\nEhITE5GUlISkpAe597rniYmJ+vcL3sfEXEdKSnKZarOzs9PPmjnLLhuGroXJyABOn87bTawL2mvX\nDI/BBgaKeO65HDRtqgvZhg21PGuYqAoTBAFOTk5wcnKCv39Amb9eq9UiOTnJIKSTkhILBHhigZth\nkF+/fg05OWX7j72rq5s+gH18vGBraw9HR139jo6OcHJy1t87ORV87lRgXP69s7Pu3hrCnKErI60W\niI5W4b//VPpZ7PnzKmg0+bttPD0ldOyoyQ1YLZo0EeHrK8lYNREpjVqthqenFzw9vcr8tWWdZRcM\n7piY6zh79rTJ+rC3ty8S2oXDOu/2sJA3DHvDcLezs6vwbnWGrplIEnD7tqA/Bnv8uBonTqiRlpb/\nA7S3l9CkiW43cdOmulvt2hJ46ISILFVFZ9ne3s6IibmH9PR0ZGSkF3ufd8vIyEB6elqh+4LjdK+l\npaUjOTkZsbGxyMhIhyia5jKyarW6SFjnzcT3799r1DZKDd2ZM2di79698Pb2xu7du0sbTrmSkqDf\nRZx3NnHBKzgJgoSICBFNm4r6WWzduiLs7GQsmojIzFQqFZydneFcScfIJElCVlZWgSDXBXZ6enEB\nXtmry4cAAAsRSURBVFyQFwx9w/vExESkp6eVafd6qaHbp08fvPjii5g2bVqFGleyrCzg7FmVwdnE\nly4ZHluoXl1E9+45aNpUN5Nt3FjLz8ASEVUyQRDg4OAABweHcu0+N4ZJQ7dFixa4detWhQpSElEE\nLl/WHYfNm8meOaNCTk7+PmBXV91C6rrdxLqZrL8/j8MSESlRWS7vyWO6pbh7V3ccNu9kpxMn1EhJ\nyQ9YW1sJDRqI+mOwzZrplqZTFb3oExERVXEM3QIkCTh/XoUDB9Q4fhw4csS5yPWIw8K06NYt/2Sn\nRx4xXPuViIjoYSotdH19reOA5Y0bwG+/6W6//w7cvZv/np+fCj17Ao8+CrRqBbRoAXh4qAGoAVjP\naiHW8rMoiRJ6ANiHJVFCD4Ay+lBCD8YyKnQlqezHI+PiUsr8NeaQlAQcPGiD/fvV2L/fBpcv589k\nq1UT0a+fFu3aadCzpyMcHVMMPq6TkwPExclQdAXkXbHGmimhB4B9WBIl9AAoow8l9AAY/x+HUkN3\nypQpOHr0KBITE9GhQweMGzcOffv2rXCB5pKVBfzzj1ofsidOqCCKuiR1dpbQpYsG7dpp0K6d7tKJ\neSHr62t9AUtERJat1NBdunSpOeowGVHUfXxn3z5dyB49mr8QgI2NbhGAdu10t2bNtOCa0kREZC6K\nOJHq+nUB+/frdhkfOKBGQkL+LuN69fJCVoPHH9dy8XUiIpKNVYZuQoLuuGzebPb69fyQrV5dxMCB\nOWjXToMnntDCz4+fjyUiIstgFaGbkQEcPZp/XPb0aRUkSbfL2M1NwjPP5Ohns6GhvFYxERFZJosM\nXa0WOHVKpd9l/PffamRl6ZLUzk5Cmzb5u4wbNdKtG0tERGTpLCKuJAm4elXAvn26kD140AZJSfnT\n1YYN80O2VSstnJxkLJaIiKicZAvde/cEHDyYv8v45s3847KBgSJ69tTtMm7TRgsfHx6XJSIi62e2\n0E1NBY4cUetns+fP56/C4+kp6UO2XTsNgoMZskREpDyVFro5OcDx4/nHZf/9Vw2NRrfL2MFBQvv2\nugtStG+vQYMGXCCAiIiUr1JCt2dP4M8/XZCaqgtZQZDQpImov/JTy5ZaODhUxncmIiKyXJUSurt3\nAyEhEvr10+0ybttWAw+PyvhORERE1qNSQvfaNcDJKa0yNk1ERGS1KuVIalBQZWyViIjIuvH0JSIi\nIjNh6BIREZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIR\nEZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eI\niMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMGLpE\nRERmwtAlIiIyE4YuERGRmRgVuvv370e3bt3QtWtXfPbZZ5VdExERkSKVGrqiKGLevHlYvXo1vv/+\ne/zwww+4fPmyOWojIiJSlFJD99SpUwgKCkKNGjVga2uL7t274/fffzdHbURERIpSaujevXsXAQEB\n+ud+fn64d+9epRZFRESkRKWGriRJ5qiDiIhI8WxKG+Dv74/bt2/rn9+9exfVqlUrdcO+vq4Vq8wC\nKKEHQBl9KKEHgH1YEiX0ACijDyX0YKxSZ7oNGzZETEwMbt26hezsbPzwww/o1KmTOWojIiJSlFJn\numq1GrNnz8bw4cMhSRL69euH0NBQc9RGRESkKILEg7ZERERmwStSERERmQlDl4iIyEwYukRERGZS\n6olUZbF//34sWLAAkiShb9++eOWVV0y5ebOYOXMm9u7dC29vb+zevVvucsolNjYW06ZNQ3x8PNRq\nNfr3748hQ4bIXVaZZWdnIzIyEjk5OdBqtejatSvGjh0rd1nlIooi+vbtCz8/P6xatUrucsqlY8eO\ncHFxgUqlgo2NDbZs2SJ3SeWSkpKCN998E9HR0VCpVFiwYAEaN24sd1lGu3r1KiZNmgRBECBJEm7c\nuIEJEyZY5d/xr7/+Glu2bIEgCKhTpw4W/r+9u3mJag8DOP6dHKRQexElCyzIjCySFr1AEyamSTXV\nxGCLNiVRbdIow14oghYJLfoHWkREEBEaRG1EszGmQiuGYIgwIhhMKkRT5yXPnOcu4l64G+89x7nz\na7rPZz1n+A6HmYcznHmmo4P8/HzTWY7cunXrr/fCv/qslQxJp9NSX18vsVhMfvz4IXv37pWhoaFM\nPX3WDAwMSDQaFb/fbzrFtS9fvkg0GhURkcnJSdmxY0dOngsRkXg8LiIilmVJU1OTRCIRw0Xu3Lx5\nU9ra2uT48eOmU1yrq6uTsbEx0xmzdvbsWbl//76IiExPT8vExIThIvfS6bT4fD4ZHh42neLYyMiI\n1NXVSSqVEhGRkydPSldXl+EqZ96/fy9+v19SqZRYliWHDx+WT58+zXhMxr5e/l12NG/YsIH58+eb\nzpiV0tJSqqqqACgoKKCioiJnV3fOmzcP+HnVa1mW4Rp3RkZGePr0KU1NTaZTZkVEsG3bdMasTE5O\nMjg4SDAYBMDr9VJYWGi4yr1wOMyyZcv+tqo3l9i2TSKRwLIsksnkv1q89Cv58OED69evJz8/n7y8\nPDZu3Eh3d/eMx2Rs6OqO5l9TLBbj3bt3VFdXm05xxbZtAoEAPp8Pn8+Xk6/j6tWrtLe34/F4TKfM\nisfj4ciRIwSDQe7du2c6x5VYLMaiRYs4f/48+/fv59KlSySTSdNZrj1+/Jjdu3ebznBl8eLFNDc3\nU1tbS01NDUVFRWzZssV0liOVlZUMDAwwPj5OIpEgFArx+fPnGY/J2NAV/bnvL2dqaorW1lYuXLhA\nQUGB6RxX5syZw4MHDwiFQkQiEYaGhkwnOdLX10dJSQlVVVU5/x65e/cunZ2d3Lhxgzt37jA4OGg6\nyTHLsohGoxw8eJCuri7mzp2bs/8RPj09TW9vLzt37jSd4sr379/p6enhyZMn9Pf3E4/Hc+4+moqK\nCo4ePUpzczPHjh1j9erVeL0z3yqVsaHrdkez+m9YlkVrayv79u2jvr7edM6sFRYWsmnTJvr7+02n\nOPL69Wt6e3vZvn07bW1tvHz5kvb2dtNZrpSWlgJQXFxMQ0MDb9++NVzkXFlZGWVlZaxbtw6AxsZG\notGo4Sp3QqEQa9eupbi42HSKK+FwmPLychYuXEheXh4NDQ28efPGdJZjwWCQzs5Obt++zYIFC1i+\nfPmMj8/Y0P2ddjTn+hUJ/LwLe+XKlRw6dMh0imujo6NMTEwAkEwmef78OStWrDBc5czp06fp6+uj\np6eH69evs3nzZq5du2Y6y7FEIsHU1BQA8XicZ8+eUVlZabjKuZKSEpYsWcLHjx8BePHiRc6utX30\n6BF+v990hmtLly4lEomQSqUQkZw9F6OjowAMDw/T3d39j+ckYz8Z+l12NP95NTI2NkZtbS0tLS1/\n3XSRK169esXDhw9ZtWoVgUAAj8fDqVOnqKmpMZ3myNevXzl37hy2bWPbNrt27WLbtm2ms/6Xvn37\nxokTJ/B4PKTTafbs2cPWrVtNZ7ly8eJFzpw5g2VZlJeX09HRYTrJsWQySTgc5sqVK6ZTXKuurqax\nsZFAIIDX62XNmjUcOHDAdJZjLS0tjI+P4/V6uXz5MkVFM/9jku5eVkoppbJEN1IppZRSWaJDVyml\nlMoSHbpKKaVUlujQVUoppbJEh65SSimVJTp0lVJKqSzRoauUUkpliQ5dpZRSKkv+AO2e4yf8wTuC\nAAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0xc1dc310\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# Train our variables.\n", - "\n", - "# numpy is used for its asscalar() function.\n", - "import numpy as np\n", - "\n", - "num_training_steps = 10\n", - "\n", - "def train_model(inputs, labels, wb, optimizer, num_training_steps):\n", - " loss_at_step = []\n", - " w_at_step = []\n", - " b_at_step = []\n", - " for step_num in range(num_training_steps):\n", - " loss, gradients_and_variables = value_and_gradients_fn(inputs, labels, wb)\n", - " loss_at_step.append(np.asscalar(loss.numpy()))\n", - " \n", - " optimizer.apply_gradients(gradients_and_variables)\n", - " w, b = wb.variables\n", - " w_at_step.append(np.asscalar(w.read_value().numpy()))\n", - " b_at_step.append(np.asscalar(b.read_value().numpy()))\n", - "\n", - " print(w_at_step)\n", - " t = range(0, num_training_steps)\n", - " plt.plot(t, loss_at_step, 'k',\n", - " t, w_at_step, 'r',\n", - " t, [true_w] * num_training_steps, 'r--',\n", - " t, b_at_step, 'b',\n", - " t, [true_b] * num_training_steps, 'b--')\n", - " plt.legend(['loss', 'w estimate', 'w true', 'b estimate', 'b true'])\n", - " plt.show()\n", - "\n", - "train_model(inputs, labels, wb, optimizer, num_training_steps)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UNurY9VJ-hpH" - }, - "source": [ - "## Other Ways to Compute Gradients\n", - "\n", - "Using our loss function as an example (`calculate_linear_model_loss()`), there are several other ways we could compute gradients:\n", - "\n", - "1. `tfe.implicit_gradients()`\n", - "1. `tfe.gradients_function()`\n", - "1. `tfe.implicit_value_and_gradients()`\n", - "1. `tfe.value_and_gradients_function()`\n", - "\n", - "Each of these functions does the following:\n", - "* Wraps a function.\n", - "* Returns a function with the same input signature as the wrapped function.\n", - "\n", - "They differ only in what information they return.\n", - "\n", - "### Gradients-only functions\n", - "\n", - "The following two functions return a function that returns only the variables' gradients:\n", - "\n", - "1. `tfe.gradients_function()`: Returns the partial derivatives of the function `f()` with respect to the parameters of `f()`.\n", - "1. `tfe.implicit_gradients()`: Returns the partial derivatives of the function `f()` with respect to the trainable parameters (`tf.Variable`) used by `f()`.\n", - "\n", - "In our example above, the `tf.layers.Dense` object encapsulates the trainable parameters.\n", - "\n", - "### Value and gradients functions\n", - "\n", - "The following two functions are identical to their counterparts above, except that they also return the value of the wrapped function.\n", - "\n", - "1. `tfe.implicit_value_and_gradients()`\n", - "1. `tfe.value_and_gradients_function()`\n", - "\n", - "### Gradient demos\n", - "\n", - "In the demos below, we show examples for the `implicit_*` functions, since our existing loss function works seamlessly with these versions. (The other versions require that your parameters are tensors and tensors only; in our example, we're using a `Dense` layer.)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 85, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 100, - "status": "ok", - "timestamp": 1505502831671, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "aEoCftnfAIH5", - "outputId": "72f1c1dc-a574-463f-f860-c4e5f48fcdaa" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[(\u003ctf.Tensor: id=673, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", - " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", - " (\u003ctf.Tensor: id=671, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", - " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)]" - ] - }, - "execution_count": 13, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# tfe.implicit_gradients() demo\n", - "gradients_fn = tfe.implicit_gradients(loss_fn)\n", - "\n", - "# Returns only gradients and variables:\n", - "gradients_fn(inputs, labels, wb)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 102, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 88, - "status": "ok", - "timestamp": 1505502831785, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "bbgCUdCzAVhH", - "outputId": "152aa9b6-9e42-4b7e-848a-9423c0b1929c" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(\u003ctf.Tensor: id=688, shape=(), dtype=float32, numpy=1.0623235\u003e,\n", - " [(\u003ctf.Tensor: id=720, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", - " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", - " (\u003ctf.Tensor: id=718, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", - " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)])" - ] - }, - "execution_count": 14, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# tfe.implicit_value_and_gradients() demo\n", - "value_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)\n", - "\n", - "# Returns only gradients:\n", - "value_gradients_fn(inputs, labels, wb)" - ] - } - ], - "metadata": { - "colab": { - "default_view": {}, - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "Eager Execution Tutorial: Working with Gradients", - "provenance": [], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb deleted file mode 100644 index ff0ff4a6a7..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb +++ /dev/null @@ -1,218 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# Eager Execution Tutorial: Importing Data\n", - "\n", - "This notebook demonstrates the use of the [`tf.contrib.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n", - "\n", - "* Creating a `Dataset`.\n", - "* Iteration over a `Dataset` with eager execution enabled.\n", - "\n", - "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n", - "\n", - "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly different. You will use a Pythonic `Iterator()` class instead of using `make_one_shot_iterator()` and `get_next()`. As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "# Setup: Enable eager execution\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RlIWhyeLoYnG" - }, - "outputs": [], - "source": [ - "# Import TensorFlow.\n", - "import tensorflow as tf\n", - "\n", - "# Import TensorFlow eager execution support (subject to future changes).\n", - "from tensorflow.contrib.eager.python import tfe\n", - "\n", - "# Enable eager execution\n", - "tfe.enable_eager_execution()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "# Step 1: Create a source `Dataset`\n", - "\n", - "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "WPTUfGq6kJ5w" - }, - "outputs": [], - "source": [ - "ds_tensors = tf.contrib.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", - "\n", - "# Create a CSV file\n", - "import tempfile\n", - "_, filename = tempfile.mkstemp()\n", - "with open(filename, 'w') as f:\n", - " f.write(\"\"\"Line 1\n", - "Line 2\n", - "Line 3\n", - " \"\"\")\n", - "ds_file = tf.contrib.data.TextLineDataset(filename)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "twBfWd5xyu_d" - }, - "source": [ - "# Step 2: Apply transformations\n", - "\n", - "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.contrib.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset) for details." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ngUe237Wt48W" - }, - "outputs": [], - "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", - "ds_file = ds_file.batch(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "# Step 3: Iterate\n", - "\n", - "Use `tfe.Iterator` on the `Dataset` object to get a Python iterator over the contents of the dataset.\n", - "\n", - "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that this process of iteration is different. Here there are no calls to `Dataset.make_one_shot_iterator()` and no `get_next()` calls." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 153, - "output_extras": [ - { - "item_id": 1 - } - ] - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 201, - "status": "ok", - "timestamp": 1505952405928, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "lCUWzso6mbqR", - "outputId": "ec027d30-96c6-4ea4-9ee1-ef74ec1ae29a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Elements of ds_tensors:\n", - "tf.Tensor([4 9], shape=(2,), dtype=int32)\n", - "tf.Tensor([16 25], shape=(2,), dtype=int32)\n", - "tf.Tensor([36 1], shape=(2,), dtype=int32)\n", - "\n", - "Elements in ds_file:\n", - "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n", - "tf.Tensor(['Line 3' ' '], shape=(2,), dtype=string)\n" - ] - } - ], - "source": [ - "print('Elements of ds_tensors:')\n", - "for x in tfe.Iterator(ds_tensors):\n", - " print(x)\n", - "\n", - "print('\\nElements in ds_file:')\n", - "for x in tfe.Iterator(ds_file):\n", - " print(x)" - ] - } - ], - "metadata": { - "colab": { - "default_view": {}, - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "Eager Execution Tutorial: Importing Data", - "provenance": [], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py deleted file mode 100644 index 7a213e9e03..0000000000 --- a/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.eager.python.examples import cart_pole_helper -from tensorflow.python.framework import test_util -from tensorflow.python.platform import test - - -class RewardDiscountingTest(test_util.TensorFlowTestCase): - - def testDiscountingRewards(self): - rewards = [0.0, 10.0, 20.0] - discount_rate = 0.9 - self.assertAllClose( - [10 * discount_rate + 20 * discount_rate * discount_rate, - 10 + 20 * discount_rate, 20], - cart_pole_helper.discount_rewards(rewards, discount_rate)) - self.assertAllClose( - [-1.2], cart_pole_helper.discount_rewards([-1.2], discount_rate)) - self.assertEqual([], cart_pole_helper.discount_rewards([], discount_rate)) - - def testDiscountAndNormalizeRewardSequences(self): - rewards1 = [0.0, 10.0, 20.0] - rewards2 = [0.0, 5.0, -5.0] - reward_sequences = [rewards1, rewards2] - discount_rate = 0.9 - dn = cart_pole_helper.discount_and_normalize_rewards(reward_sequences, - discount_rate) - self.assertAllClose( - [[1.03494653, 1.24685514, 0.64140196], - [-0.83817424, -0.83439016, -1.25063922]], dn) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py deleted file mode 100644 index dc1381cc04..0000000000 --- a/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit test for cart-pole reinforcement learning under eager exection.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gc -import glob -import os -import shutil -import tempfile -import time - -import gym -import numpy as np - -from tensorflow.contrib.eager.python.examples import cart_pole -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import training - - -class CartPoleTest(test_util.TensorFlowTestCase): - - def setUp(self): - super(CartPoleTest, self).setUp() - self._tmp_logdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self._tmp_logdir) - super(CartPoleTest, self).tearDown() - - def testGetLogitsAndAction(self): - hidden_size = 5 - policy_network = cart_pole.PolicyNetwork(hidden_size) - - dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5], - [0.0, -0.2, 0.6, -0.8]], dtype=np.float32) - logits, actions = policy_network.forward(constant_op.constant(dummy_inputs)) - - self.assertEqual((2, 1), logits.shape) - self.assertEqual(dtypes.float32, logits.dtype) - self.assertEqual((2, 1), actions.shape) - self.assertEqual(dtypes.int64, actions.dtype) - - def testCrossEntropy(self): - hidden_size = 5 - policy_network = cart_pole.PolicyNetwork(hidden_size) - - dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5], - [0.0, -0.2, 0.6, -0.8]], dtype=np.float32) - cross_entropy = policy_network._get_cross_entropy_and_save_actions( - constant_op.constant(dummy_inputs)) - - self.assertEqual((2, 1), cross_entropy.shape) - self.assertEqual(dtypes.float32, cross_entropy.dtype) - - def testPlayAGame(self): - hidden_size = 5 - cart_pole_env = gym.make("CartPole-v0") - cart_pole_env.seed(0) - cart_pole_env.reset() - - device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" - logging.info("device = %s", device) - with context.device(device): - policy_network = cart_pole.PolicyNetwork(hidden_size) - policy_network.play(cart_pole_env, max_steps=10, render=False) - - def testTrain(self): - hidden_size = 5 - num_games_per_iteration = 5 - max_steps_per_game = 10 - discount_rate = 0.95 - learning_rate = 0.02 - - cart_pole_env = gym.make("CartPole-v0") - cart_pole_env.reset() - - device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" - logging.info("device = %s", device) - with context.device(device): - policy_network = cart_pole.PolicyNetwork(hidden_size, - train_logdir=self._tmp_logdir) - optimizer = training.AdamOptimizer(learning_rate) - policy_network.train( - cart_pole_env, - optimizer, - discount_rate, - num_games_per_iteration, - max_steps_per_game) - self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) - - -class EagerCartPoleTrainingBenchmark(test.Benchmark): - - def benchmarkEagerCartPolePolicyNetworkTraining(self): - burn_in_iterations = 1 - benchmark_iterations = 2 - num_games_per_iteration = 10 - max_steps_per_game = 100 - discount_rate = 0.95 - learning_rate = 0.02 - - cart_pole_env = gym.make("CartPole-v0") - cart_pole_env.seed(0) - random_seed.set_random_seed(0) - cart_pole_env.reset() - - hidden_size = 5 - policy_network = cart_pole.PolicyNetwork(hidden_size) - optimizer = training.AdamOptimizer(learning_rate) - - # Perform burn-in. - for _ in xrange(burn_in_iterations): - policy_network.train( - cart_pole_env, - optimizer, - discount_rate, - num_games_per_iteration, - max_steps_per_game) - - gc.collect() - start_time = time.time() - for _ in xrange(benchmark_iterations): - policy_network.train( - cart_pole_env, - optimizer, - discount_rate, - num_games_per_iteration, - max_steps_per_game) - wall_time = time.time() - start_time - # Named "examples"_per_sec to conform with other benchmarks. - extras = {"examples_per_sec": benchmark_iterations / wall_time} - self.report_benchmark( - name="EagerCartPoleReinforcementLearning", - iters=benchmark_iterations, - wall_time=wall_time, - extras=extras) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py deleted file mode 100644 index aee0b3d0dd..0000000000 --- a/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Unit tests for linear regression example under TensorFlow eager execution.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import glob -import os -import shutil -import tempfile -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python.examples import linear_regression -from tensorflow.python.eager import context -from tensorflow.python.eager import test -from tensorflow.python.framework import test_util -from tensorflow.python.platform import tf_logging as logging - - -def _create_data_gen_for_test(): - true_w = np.array([[1.0], [-0.5], [2.0]], dtype=np.float32) - true_b = np.array([1.0], dtype=np.float32) - noise_level = 0 - batch_size = 64 - return ( - true_w, true_b, noise_level, batch_size, - linear_regression.DataGenerator(true_w, true_b, noise_level, batch_size)) - - -class LinearRegressionTest(test_util.TensorFlowTestCase): - - def setUp(self): - super(LinearRegressionTest, self).setUp() - self._tmp_logdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self._tmp_logdir) - super(LinearRegressionTest, self).tearDown() - - def testSyntheticBatch(self): - _, _, _, batch_size, data_gen = _create_data_gen_for_test() - - xs, ys = data_gen.next_batch() - self.assertEqual((batch_size, 3), xs.shape) - self.assertEqual((batch_size, 1), ys.shape) - self.assertEqual(tf.float32, xs.dtype) - self.assertEqual(tf.float32, ys.dtype) - - def testLinearRegression(self): - true_w, true_b, _, _, data_gen = _create_data_gen_for_test() - - learning_rate = 0.1 - num_iters = 40 - - device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" - logging.info("device = %s", device) - with context.device(device): - linear_model = linear_regression.LinearModel() - optimizer = tf.train.GradientDescentOptimizer(learning_rate) - linear_model.fit(data_gen.next_batch, optimizer, num_iters, - logdir=self._tmp_logdir) - - self.assertAllClose(true_w, linear_model.weights, rtol=1e-2) - self.assertAllClose(true_b, linear_model.biases, rtol=1e-2) - self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) - - -class EagerLinearRegressionBenchmark(test.Benchmark): - - def benchmarkEagerLinearRegression(self): - _, _, _, _, data_gen = _create_data_gen_for_test() - - learning_rate = 0.1 - num_burnin_iters = 10 - num_iters = 200 - - device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0" - logging.info("device = %s", device) - with context.device(device): - linear_model = linear_regression.LinearModel() - optimizer = tf.train.GradientDescentOptimizer(learning_rate) - - # Perform burn-in. - linear_model.fit(data_gen.next_batch, optimizer, num_burnin_iters) - - start_time = time.time() - linear_model.fit(data_gen.next_batch, optimizer, num_iters) - wall_time = time.time() - start_time - - self.report_benchmark( - name="EagerLinearRegression", - iters=num_iters, - wall_time=wall_time) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/examples/tests/spinn_test.py b/tensorflow/contrib/eager/python/examples/tests/spinn_test.py deleted file mode 100644 index 9c8b691b98..0000000000 --- a/tensorflow/contrib/eager/python/examples/tests/spinn_test.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import gc -import time - -import numpy as np -import tensorflow as tf - -from tensorflow.contrib.eager.python import tfe -from tensorflow.contrib.eager.python.examples import spinn -from tensorflow.python.eager import test -from tensorflow.python.framework import test_util - - -def _generate_synthetic_snli_data_batch(sequence_length, - batch_size, - vocab_size): - """Generate a fake batch of SNLI data for testing.""" - with tf.device("cpu:0"): - labels = tf.random_uniform([batch_size], minval=1, maxval=4, dtype=tf.int64) - prem = tf.random_uniform( - (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64) - prem_trans = tf.constant(np.array( - [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3, - 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, - 3, 2, 2]] * batch_size, dtype=np.int64).T) - hypo = tf.random_uniform( - (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64) - hypo_trans = tf.constant(np.array( - [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3, - 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2, - 3, 2, 2]] * batch_size, dtype=np.int64).T) - if tfe.num_gpus(): - labels = labels.gpu() - prem = prem.gpu() - prem_trans = prem_trans.gpu() - hypo = hypo.gpu() - hypo_trans = hypo_trans.gpu() - return labels, prem, prem_trans, hypo, hypo_trans - - -def _snli_classifier_config(d_embed, d_out): - config_tuple = collections.namedtuple( - "Config", ["d_hidden", "d_proj", "d_tracker", "predict", - "embed_dropout", "mlp_dropout", "n_mlp_layers", "d_mlp", - "d_out", "projection", "lr"]) - config = config_tuple( - d_hidden=d_embed, - d_proj=d_embed * 2, - d_tracker=8, - predict=False, - embed_dropout=0.1, - mlp_dropout=0.1, - n_mlp_layers=2, - d_mlp=32, - d_out=d_out, - projection=True, - lr=2e-3) - return config - - -class SpinnTest(test_util.TensorFlowTestCase): - - def setUp(self): - super(SpinnTest, self).setUp() - self._test_device = "gpu:0" if tfe.num_gpus() else "cpu:0" - - def testBundle(self): - with tf.device(self._test_device): - lstm_iter = [np.array([[0, 1], [2, 3]], dtype=np.float32), - np.array([[0, -1], [-2, -3]], dtype=np.float32), - np.array([[0, 2], [4, 6]], dtype=np.float32), - np.array([[0, -2], [-4, -6]], dtype=np.float32)] - out = spinn._bundle(lstm_iter) - - self.assertEqual(2, len(out)) - self.assertEqual(tf.float32, out[0].dtype) - self.assertEqual(tf.float32, out[1].dtype) - self.assertAllEqual(np.array([[0, 2, 0, -2, 0, 4, 0, -4]]).T, - out[0].numpy()) - self.assertAllEqual(np.array([[1, 3, -1, -3, 2, 6, -2, -6]]).T, - out[1].numpy()) - - def testUnbunbdle(self): - with tf.device(self._test_device): - state = [np.array([[0, 1, 2], [3, 4, 5]], dtype=np.float32), - np.array([[0, -1, -2], [-3, -4, -5]], dtype=np.float32)] - out = spinn._unbundle(state) - - self.assertEqual(2, len(out)) - self.assertEqual(tf.float32, out[0].dtype) - self.assertEqual(tf.float32, out[1].dtype) - self.assertAllEqual(np.array([[0, 1, 2, 0, -1, -2]]), - out[0].numpy()) - self.assertAllEqual(np.array([[3, 4, 5, -3, -4, -5]]), - out[1].numpy()) - - def testReduce(self): - with tf.device(self._test_device): - batch_size = 3 - size = 10 - tracker_size = 8 - reducer = spinn.Reduce(size, tracker_size=tracker_size) - - left_in = [] - right_in = [] - tracking = [] - for _ in range(batch_size): - left_in.append(tf.random_normal((1, size * 2))) - right_in.append(tf.random_normal((1, size * 2))) - tracking.append(tf.random_normal((1, tracker_size * 2))) - - out = reducer(left_in, right_in, tracking=tracking) - self.assertEqual(batch_size, len(out)) - self.assertEqual(tf.float32, out[0].dtype) - self.assertEqual((1, size * 2), out[0].shape) - - def testReduceTreeLSTM(self): - with tf.device(self._test_device): - size = 10 - tracker_size = 8 - reducer = spinn.Reduce(size, tracker_size=tracker_size) - - lstm_in = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]], - dtype=np.float32) - c1 = np.array([[0, 1], [2, 3]], dtype=np.float32) - c2 = np.array([[0, -1], [-2, -3]], dtype=np.float32) - - h, c = reducer._tree_lstm(c1, c2, lstm_in) - self.assertEqual(tf.float32, h.dtype) - self.assertEqual(tf.float32, c.dtype) - self.assertEqual((2, 2), h.shape) - self.assertEqual((2, 2), c.shape) - - def testTracker(self): - with tf.device(self._test_device): - batch_size = 2 - size = 10 - tracker_size = 8 - buffer_length = 18 - stack_size = 3 - - tracker = spinn.Tracker(tracker_size, False) - tracker.reset_state() - - # Create dummy inputs for testing. - bufs = [] - buf = [] - for _ in range(buffer_length): - buf.append(tf.random_normal((batch_size, size * 2))) - bufs.append(buf) - self.assertEqual(1, len(bufs)) - self.assertEqual(buffer_length, len(bufs[0])) - self.assertEqual((batch_size, size * 2), bufs[0][0].shape) - - stacks = [] - stack = [] - for _ in range(stack_size): - stack.append(tf.random_normal((batch_size, size * 2))) - stacks.append(stack) - self.assertEqual(1, len(stacks)) - self.assertEqual(3, len(stacks[0])) - self.assertEqual((batch_size, size * 2), stacks[0][0].shape) - - for _ in range(2): - out1, out2 = tracker(bufs, stacks) - self.assertIsNone(out2) - self.assertEqual(batch_size, len(out1)) - self.assertEqual(tf.float32, out1[0].dtype) - self.assertEqual((1, tracker_size * 2), out1[0].shape) - - self.assertEqual(tf.float32, tracker.state.c.dtype) - self.assertEqual((batch_size, tracker_size), tracker.state.c.shape) - self.assertEqual(tf.float32, tracker.state.h.dtype) - self.assertEqual((batch_size, tracker_size), tracker.state.h.shape) - - def testSPINN(self): - with tf.device(self._test_device): - embedding_dims = 10 - d_tracker = 8 - sequence_length = 15 - num_transitions = 27 - - config_tuple = collections.namedtuple( - "Config", ["d_hidden", "d_proj", "d_tracker", "predict"]) - config = config_tuple( - embedding_dims, embedding_dims * 2, d_tracker, False) - s = spinn.SPINN(config) - - # Create some fake data. - buffers = tf.random_normal((sequence_length, 1, config.d_proj)) - transitions = np.array( - [[3], [3], [2], [3], [3], [3], [2], [2], [2], [3], [3], [3], - [2], [3], [3], [2], [2], [3], [3], [3], [2], [2], [2], [2], - [3], [2], [2]], dtype=np.int32) - self.assertEqual(tf.int32, transitions.dtype) - self.assertEqual((num_transitions, 1), transitions.shape) - - out = s(buffers, transitions, training=True) - self.assertEqual(tf.float32, out.dtype) - self.assertEqual((1, embedding_dims), out.shape) - - def testSNLIClassifierAndTrainer(self): - with tf.device(self._test_device): - vocab_size = 40 - batch_size = 2 - d_embed = 10 - sequence_length = 15 - d_out = 4 - - config = _snli_classifier_config(d_embed, d_out) - - # Create fake embedding matrix. - embed = tf.random_normal((vocab_size, d_embed)) - - model = spinn.SNLIClassifier(config, embed) - trainer = spinn.SNLIClassifierTrainer(model, config.lr) - - (labels, prem, prem_trans, hypo, - hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length, - batch_size, - vocab_size) - - # Invoke model under non-training mode. - logits = model(prem, prem_trans, hypo, hypo_trans, training=False) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((batch_size, d_out), logits.shape) - - # Invoke model under training model. - logits = model(prem, prem_trans, hypo, hypo_trans, training=True) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((batch_size, d_out), logits.shape) - - # Calculate loss. - loss1 = trainer.loss(labels, logits) - self.assertEqual(tf.float32, loss1.dtype) - self.assertEqual((), loss1.shape) - - loss2, logits = trainer.train_batch( - labels, prem, prem_trans, hypo, hypo_trans) - self.assertEqual(tf.float32, loss2.dtype) - self.assertEqual((), loss2.shape) - self.assertEqual(tf.float32, logits.dtype) - self.assertEqual((batch_size, d_out), logits.shape) - # Training on the batch should have led to a change in the loss value. - self.assertNotEqual(loss1.numpy(), loss2.numpy()) - - -class EagerSpinnSNLIClassifierBenchmark(test.Benchmark): - - def benchmarkEagerSpinnSNLIClassifier(self): - test_device = "gpu:0" if tfe.num_gpus() else "cpu:0" - with tf.device(test_device): - burn_in_iterations = 2 - benchmark_iterations = 10 - - vocab_size = 1000 - batch_size = 128 - sequence_length = 15 - d_embed = 200 - d_out = 4 - - embed = tf.random_normal((vocab_size, d_embed)) - - config = _snli_classifier_config(d_embed, d_out) - model = spinn.SNLIClassifier(config, embed) - trainer = spinn.SNLIClassifierTrainer(model, config.lr) - - (labels, prem, prem_trans, hypo, - hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length, - batch_size, - vocab_size) - - for _ in range(burn_in_iterations): - trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans) - - gc.collect() - start_time = time.time() - for _ in xrange(benchmark_iterations): - trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans) - wall_time = time.time() - start_time - # Named "examples"_per_sec to conform with other benchmarks. - extras = {"examples_per_sec": benchmark_iterations / wall_time} - self.report_benchmark( - name="Eager_SPINN_SNLIClassifier_Benchmark", - iters=benchmark_iterations, - wall_time=wall_time, - extras=extras) - - -if __name__ == "__main__": - test.main() diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 4ed258f6ff..3810d96950 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -55,6 +55,10 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@IsolateTest @@run_test_in_graph_and_eager_modes + +@@DEVICE_PLACEMENT_EXPLICIT +@@DEVICE_PLACEMENT_WARN +@@DEVICE_PLACEMENT_SILENT """ from __future__ import absolute_import @@ -71,6 +75,9 @@ from tensorflow.contrib.eager.python.saver import Saver from tensorflow.contrib.eager.python.summary_writer import SummaryWriter from tensorflow.python.eager import backprop from tensorflow.python.eager import function +from tensorflow.python.eager.context import DEVICE_PLACEMENT_EXPLICIT +from tensorflow.python.eager.context import DEVICE_PLACEMENT_WARN +from tensorflow.python.eager.context import DEVICE_PLACEMENT_SILENT from tensorflow.python.eager.context import in_eager_mode from tensorflow.python.eager.context import in_graph_mode from tensorflow.python.eager.context import list_devices diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 58581283d2..c5eedb7c9c 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -42,6 +42,10 @@ _device_parsing_cache = {} _MAXINT32 = 2**31 - 1 +DEVICE_PLACEMENT_EXPLICIT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_EXPLICIT +DEVICE_PLACEMENT_WARN = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_WARN +DEVICE_PLACEMENT_SILENT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_SILENT + # TODO(agarwal): better name ? class _EagerContext(threading.local): @@ -62,13 +66,22 @@ class _EagerContext(threading.local): class Context(object): """Environment in which eager operations execute.""" - def __init__(self, config=None): + def __init__(self, config=None, device_policy=None): """Creates a new Context. Args: config: (Optional.) A `ConfigProto` protocol buffer with configuration - options for the Context. Note that a lot of these options may be - currently unimplemented or irrelevant for EAGER mode. + options for the Context. Note that a lot of these options may be + currently unimplemented or irrelevant when eager execution is enabled. + device_policy: (Optional.) What policy to use when trying to run an + operation on a device with inputs which are not on that device. + Valid values: + tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not + correct. + tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the + right device but raises a warning. + tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might + hide performance problems. """ self._eager_context = _EagerContext() self._context_handle = None @@ -78,6 +91,7 @@ class Context(object): self._config = config self._seed = None self._initialize_lock = threading.Lock() + self._device_policy = device_policy def _set_global_seed(self, seed): """Set a global eager mode seed for random ops.""" @@ -109,6 +123,9 @@ class Context(object): config_str = self._config.SerializeToString() pywrap_tensorflow.TFE_ContextOptionsSetConfig( opts, config_str, len(config_str), status) + if self._device_policy is not None: + pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy( + opts, self._device_policy) self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status) finally: pywrap_tensorflow.TFE_DeleteContextOptions(opts) diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 2ebb625f9f..1cd3826755 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -266,6 +266,23 @@ class OpsTest(test_util.TensorFlowTestCase): shape = array_ops.shape(value) self.assertEqual([1], shape.numpy()) + def testSilentCopy(self): + if not context.context().num_gpus(): + self.skipTest('No GPUs found') + # Temporarily replace the context + # pylint: disable=protected-access + del context._context + try: + context._context = context.Context( + device_policy=context.DEVICE_PLACEMENT_SILENT) + cpu_tensor = constant_op.constant(1.0) + gpu_tensor = cpu_tensor.gpu() + self.assertAllEqual(cpu_tensor + gpu_tensor, 2.0) + finally: + del context._context + context._context = context.Context() + # pylint: enable=protected-access + def testRandomUniform(self): scalar_shape = constant_op.constant([], dtype=dtypes.int32) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index ef708a4703..b45cb2e0c6 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -4559,7 +4559,7 @@ class _DefaultGraphStack(_DefaultStack): # pylint: disable=protected-access _default_graph_stack = _DefaultGraphStack() -def enable_eager_execution(): +def enable_eager_execution(config=None, device_policy=None): """Enables, for the rest of the lifetime of this program, eager execution. If not called immediately on startup risks creating breakage and bugs. @@ -4574,8 +4574,24 @@ def enable_eager_execution(): assert tf.multiply(6, 7).numpy() == 42 ``` + Args: + config: (Optional.) A `ConfigProto` protocol buffer with configuration + options for the Context. Note that a lot of these options may be + currently unimplemented or irrelevant when eager execution is enabled. + device_policy: (Optional.) What policy to use when trying to run an + operation on a device with inputs which are not on that device. + Valid values: + tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not + correct. + tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the + right device but raises a warning. + tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might + hide performance problems. + Raises: - ValueError: If this method has already been invoked in the current process. + ValueError: If trying to create a context after using graph operations + or if trying to create a context with nontrivial options which differ + from those of the existing context. """ # pylint: disable=protected-access if context._default_mode == context.GRAPH_MODE: @@ -4586,6 +4602,18 @@ def enable_eager_execution(): raise ValueError( "tfe.enable_eager_execution has to be called at program startup.") context._default_mode = context.EAGER_MODE + if context._context is None: + context._context = context.Context(config=config, + device_policy=device_policy) + elif ((config is not None and config is not context._context._config) + or (device_policy is not None + and device_policy is not context._context._device_policy)): + raise ValueError("Trying to change the options of an active eager" + " execution. Context config: %s, specified config:" + " %s. Context device policy: %s; specified device" + " policy: %s." % (config, context._context._config, + device_policy, + context._context._device_policy)) def eager_run(main=None, argv=None): diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 36c09c20c2..fa36b77311 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -32,6 +32,7 @@ limitations under the License. %rename("%s") TFE_Py_TapeExport; %rename("%s") TFE_NewContextOptions; %rename("%s") TFE_ContextOptionsSetConfig; +%rename("%s") TFE_ContextOptionsSetDevicePlacementPolicy; %rename("%s") TFE_DeleteContextOptions; %{ @@ -101,6 +102,11 @@ limitations under the License. } } +%rename("%s") TFE_ContextDevicePlacementPolicy; +%rename("%s") TFE_DEVICE_PLACEMENT_EXPLICIT; +%rename("%s") TFE_DEVICE_PLACEMENT_WARN; +%rename("%s") TFE_DEVICE_PLACEMENT_SILENT; + %include "tensorflow/c/eager/c_api.h" %typemap(in) TFE_InputTensorHandles* inputs (TFE_InputTensorHandles temp) { -- GitLab From ccfd9c1e50934e4b16d40d4d5d87206ad871996d Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Fri, 20 Oct 2017 15:40:34 -0700 Subject: [PATCH 225/573] Log Hlo IR during AOT compilation PiperOrigin-RevId: 172944165 --- tensorflow/compiler/xla/service/cpu/cpu_compiler.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 06e7ec0c7c..99b5035c2d 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -757,8 +757,14 @@ CpuCompiler::CompileAheadOfTime(std::vector> modules, HloModule* module = modules[i].get(); VLOG(1) << "Compiling ahead-of-time: " << module->name(); + VLOG(2) << "Before optimization:"; + XLA_VLOG_LINES(2, module->ToString()); + TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true)); + VLOG(2) << "After optimization:"; + XLA_VLOG_LINES(2, module->ToString()); + TF_ASSIGN_OR_RETURN( SequentialHloOrdering::HloModuleSequence module_sequence, CreateMemoryMinimizingSequence(*module, BufferSizeBytesFunction())); -- GitLab From ebcae4a5e3bf5c840d73a0d90f1b5bf01a68f82c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 15:55:17 -0700 Subject: [PATCH 226/573] Add streaming_precision_recall_at_equal_thresholds This helper method computes streaming tp, fp, tn, fp, precision, and recall for the user in a way that exhibits O(T + N) time and space complexity (instead of O(T * N)), where T is the number of thresholds and N is the size of the predictions tensor. Thanks to Frank Chu for the efficient algorithm! PiperOrigin-RevId: 172946073 --- .../contrib/metrics/python/ops/metric_ops.py | 180 ++++++++++++++++++ .../metrics/python/ops/metric_ops_test.py | 164 ++++++++++++++++ 2 files changed, 344 insertions(+) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 09485c4fa2..5a4c0c4358 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -22,6 +22,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections as collections_lib + from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -1076,6 +1078,9 @@ def streaming_curve_points(labels=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + + TODO(chizeng): Consider rewriting this method to make use of logic within the + streaming_precision_recall_at_equal_thresholds method (to improve run time). """ with variable_scope.variable_scope(name, 'curve_points', (labels, predictions, weights)): @@ -1193,6 +1198,181 @@ def streaming_auc(predictions, name=name) +def streaming_precision_recall_at_equal_thresholds(predictions, + labels, + num_thresholds=None, + weights=None, + name=None, + use_locking=None): + """A helper method for creating metrics related to precision-recall curves. + + These values are true positives, false negatives, true negatives, false + positives, precision, and recall. This function returns a data structure that + contains ops within it. + + Unlike _streaming_confusion_matrix_at_thresholds (which exhibits O(T * N) + space and run time), this op exhibits O(T + N) space and run time, where T is + the number of thresholds and N is the size of the predictions tensor. Hence, + it may be advantageous to use this function when `predictions` is big. + + For instance, prefer this method for per-pixel classification tasks, for which + the predictions tensor may be very large. + + Each number in `predictions`, a float in `[0, 1]`, is compared with its + corresponding label in `labels`, and counts as a single tp/fp/tn/fn value at + each threshold. This is then multiplied with `weights` which can be used to + reweight certain values, or more commonly used for masking values. + + Args: + predictions: A floating point `Tensor` of arbitrary shape and whose values + are in the range `[0, 1]`. + labels: A bool `Tensor` whose shape matches `predictions`. + num_thresholds: Optional; Number of thresholds, evenly distributed in + `[0, 1]`. Should be `>= 2`. Defaults to 201. Note that the number of bins + is 1 less than `num_thresholds`. Using an even `num_thresholds` value + instead of an odd one may yield unfriendly edges for bins. + weights: Optional; If provided, a `Tensor` that has the same dtype as, + and broadcastable to, `predictions`. This tensor is multplied by counts. + name: Optional; variable_scope name. If not provided, the string + 'precision_recall_at_equal_threshold' is used. + use_locking: Optional; If True, the op will be protected by a lock. + Otherwise, the behavior is undefined, but may exhibit less contention. + Defaults to True. + + Returns: + result: A named tuple (See PrecisionRecallData within the implementation of + this function) with properties that are variables of shape + `[num_thresholds]`. The names of the properties are tp, fp, tn, fn, + precision, recall, thresholds. + update_op: An op that accumulates values. + + Raises: + ValueError: If `predictions` and `labels` have mismatched shapes, or if + `weights` is not `None` and its shape doesn't match `predictions`, or if + `includes` contains invalid keys. + """ + # Disable the invalid-name checker so that we can capitalize the name. + # pylint: disable=invalid-name + PrecisionRecallData = collections_lib.namedtuple( + 'PrecisionRecallData', + ['tp', 'fp', 'tn', 'fn', 'precision', 'recall', 'thresholds']) + # pylint: enable=invalid-name + + if num_thresholds is None: + num_thresholds = 201 + + if weights is None: + weights = 1.0 + + if use_locking is None: + use_locking = True + + check_ops.assert_type(labels, dtypes.bool) + + dtype = predictions.dtype + with variable_scope.variable_scope(name, + 'precision_recall_at_equal_thresholds', + (labels, predictions, weights)): + # Make sure that predictions are within [0.0, 1.0]. + with ops.control_dependencies([ + check_ops.assert_greater_equal( + predictions, + math_ops.cast(0.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]'), + check_ops.assert_less_equal( + predictions, + math_ops.cast(1.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]') + ]): + predictions, labels, weights = _remove_squeezable_dimensions( + predictions=predictions, labels=labels, weights=weights) + + predictions.get_shape().assert_is_compatible_with(labels.get_shape()) + + # We cast to float to ensure we have 0.0 or 1.0. + f_labels = math_ops.cast(labels, dtype) + + # Get weighted true/false labels. + true_labels = f_labels * weights + false_labels = (1.0 - f_labels) * weights + + # Flatten predictions and labels. + predictions = array_ops.reshape(predictions, [-1]) + true_labels = array_ops.reshape(true_labels, [-1]) + false_labels = array_ops.reshape(false_labels, [-1]) + + # To compute TP/FP/TN/FN, we are measuring a binary classifier + # C(t) = (predictions >= t) + # at each threshold 't'. So we have + # TP(t) = sum( C(t) * true_labels ) + # FP(t) = sum( C(t) * false_labels ) + # + # But, computing C(t) requires computation for each t. To make it fast, + # observe that C(t) is a cumulative integral, and so if we have + # thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1} + # where n = num_thresholds, and if we can compute the bucket function + # B(i) = Sum( (predictions == t), t_i <= t < t{i+1} ) + # then we get + # C(t_i) = sum( B(j), j >= i ) + # which is the reversed cumulative sum in tf.cumsum(). + # + # We can compute B(i) efficiently by taking advantage of the fact that + # our thresholds are evenly distributed, in that + # width = 1.0 / (num_thresholds - 1) + # thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0] + # Given a prediction value p, we can map it to its bucket by + # bucket_index(p) = floor( p * (num_thresholds - 1) ) + # so we can use tf.scatter_add() to update the buckets in one pass. + # + # This implementation exhibits a run time and space complexity of O(T + N), + # where T is the number of thresholds and N is the size of predictions. + # Metrics that rely on _streaming_confusion_matrix_at_thresholds instead + # exhibit a complexity of O(T * N). + + # Compute the bucket indices for each prediction value. + bucket_indices = math_ops.cast( + math_ops.floor(predictions * (num_thresholds - 1)), dtypes.int32) + + with ops.name_scope('variables'): + tp_buckets_v = _create_local( + 'tp_buckets', shape=[num_thresholds], dtype=dtype) + fp_buckets_v = _create_local( + 'fp_buckets', shape=[num_thresholds], dtype=dtype) + + with ops.name_scope('update_op'): + update_tp = state_ops.scatter_add( + tp_buckets_v, bucket_indices, true_labels, use_locking=use_locking) + update_fp = state_ops.scatter_add( + fp_buckets_v, bucket_indices, false_labels, use_locking=use_locking) + + # Set up the cumulative sums to compute the actual metrics. + tp = math_ops.cumsum(tp_buckets_v, reverse=True, name='tp') + fp = math_ops.cumsum(fp_buckets_v, reverse=True, name='fp') + # fn = sum(true_labels) - tp + # = sum(tp_buckets) - tp + # = tp[0] - tp + # Similarly, + # tn = fp[0] - fp + tn = fp[0] - fp + fn = tp[0] - tp + + # We use a minimum to prevent division by 0. + epsilon = 1e-7 + precision = tp / math_ops.maximum(epsilon, tp + fp) + recall = tp / math_ops.maximum(epsilon, tp + fn) + + result = PrecisionRecallData( + tp=tp, + fp=fp, + tn=tn, + fn=fn, + precision=precision, + recall=recall, + thresholds=math_ops.lin_space(0.0, 1.0, num_thresholds)) + update_op = control_flow_ops.group(update_tp, update_fp) + return result, update_op + + def streaming_specificity_at_sensitivity(predictions, labels, sensitivity, diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py index f288fceef6..f24bec7f11 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py @@ -1970,6 +1970,170 @@ class StreamingAUCTest(test.TestCase): self.assertAlmostEqual(expected_auc, auc.eval(), 2) +class StreamingPrecisionRecallAtEqualThresholdsTest(test.TestCase): + + def setUp(self): + np.random.seed(1) + ops.reset_default_graph() + + def _testResultsEqual(self, expected_dict, gotten_result): + """Tests that 2 results (dicts) represent the same data. + + Args: + expected_dict: A dictionary with keys that are the names of properties + of PrecisionRecallData and whose values are lists of floats. + gotten_result: A PrecisionRecallData object. + """ + gotten_dict = {k: t.eval() for k, t in gotten_result._asdict().items()} + self.assertItemsEqual( + list(expected_dict.keys()), list(gotten_dict.keys())) + + for key, expected_values in expected_dict.items(): + self.assertAllClose(expected_values, gotten_dict[key]) + + def _testCase(self, predictions, labels, expected_result, weights=None): + """Performs a test given a certain scenario of labels, predictions, weights. + + Args: + predictions: The predictions tensor. Of type float32. + labels: The labels tensor. Of type bool. + expected_result: The expected result (dict) that maps to tensors. + weights: Optional weights tensor. + """ + with self.test_session() as sess: + predictions_tensor = constant_op.constant( + predictions, dtype=dtypes_lib.float32) + labels_tensor = constant_op.constant(labels, dtype=dtypes_lib.bool) + weights_tensor = None + if weights: + weights_tensor = constant_op.constant(weights, dtype=dtypes_lib.float32) + gotten_result, update_op = ( + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=predictions_tensor, + labels=labels_tensor, + num_thresholds=3, + weights=weights_tensor)) + + sess.run(variables.local_variables_initializer()) + sess.run(update_op) + + self._testResultsEqual(expected_result, gotten_result) + + def testVars(self): + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32), + labels=constant_op.constant([True], dtype=dtypes_lib.bool)) + _assert_local_variables( + self, + ( + 'precision_recall_at_equal_thresholds/variables/tp_buckets:0', + 'precision_recall_at_equal_thresholds/variables/fp_buckets:0' + )) + + def testVarsWithName(self): + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32), + labels=constant_op.constant([True], dtype=dtypes_lib.bool), + name='foo') + _assert_local_variables( + self, ('foo/variables/tp_buckets:0', 'foo/variables/fp_buckets:0')) + + def testValuesAreIdempotent(self): + predictions = constant_op.constant( + np.random.uniform(size=(10, 3)), dtype=dtypes_lib.float32) + labels = constant_op.constant( + np.random.uniform(size=(10, 3)) > 0.5, dtype=dtypes_lib.bool) + + result, update_op = ( + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=predictions, labels=labels)) + + with self.test_session() as sess: + # Run several updates. + sess.run(variables.local_variables_initializer()) + for _ in range(3): + sess.run(update_op) + + # Then verify idempotency. + initial_result = {k: value.eval().tolist() for k, value in + result._asdict().items()} + for _ in range(3): + self._testResultsEqual(initial_result, result) + + def testAllTruePositives(self): + self._testCase([[1]], [[True]], { + 'tp': [1, 1, 1], + 'fp': [0, 0, 0], + 'tn': [0, 0, 0], + 'fn': [0, 0, 0], + 'precision': [1.0, 1.0, 1.0], + 'recall': [1.0, 1.0, 1.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testAllTrueNegatives(self): + self._testCase([[0]], [[False]], { + 'tp': [0, 0, 0], + 'fp': [1, 0, 0], + 'tn': [0, 1, 1], + 'fn': [0, 0, 0], + 'precision': [0.0, 0.0, 0.0], + 'recall': [0.0, 0.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testAllFalsePositives(self): + self._testCase([[1]], [[False]], { + 'tp': [0, 0, 0], + 'fp': [1, 1, 1], + 'tn': [0, 0, 0], + 'fn': [0, 0, 0], + 'precision': [0.0, 0.0, 0.0], + 'recall': [0.0, 0.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testAllFalseNegatives(self): + self._testCase([[0]], [[True]], { + 'tp': [1, 0, 0], + 'fp': [0, 0, 0], + 'tn': [0, 0, 0], + 'fn': [0, 1, 1], + 'precision': [1.0, 0.0, 0.0], + 'recall': [1.0, 0.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testManyValues(self): + self._testCase( + [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], + [[True, False, False, True, True, True]], + { + 'tp': [4, 3, 0], + 'fp': [2, 0, 0], + 'tn': [0, 2, 2], + 'fn': [0, 1, 4], + 'precision': [2.0 / 3.0, 1.0, 0.0], + 'recall': [1.0, 0.75, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testManyValuesWithWeights(self): + self._testCase( + [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], + [[True, False, False, True, True, True]], + { + 'tp': [1.5, 1.5, 0.0], + 'fp': [2.5, 0.0, 0.0], + 'tn': [0.0, 2.5, 2.5], + 'fn': [0.0, 0.0, 1.5], + 'precision': [0.375, 1.0, 0.0], + 'recall': [1.0, 1.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }, + weights=[0.0, 0.5, 2.0, 0.0, 0.5, 1.0]) + + class StreamingSpecificityAtSensitivityTest(test.TestCase): def setUp(self): -- GitLab From 8ff33271ea4de89e6ff662fe8e479c1fcf56fe77 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 20 Oct 2017 15:55:57 -0700 Subject: [PATCH 227/573] Dump the computation's SessionModule as part of the tf_compile rule. PiperOrigin-RevId: 172946149 --- tensorflow/compiler/aot/compile.cc | 6 +++--- tensorflow/compiler/aot/flags.cc | 5 ++--- tensorflow/compiler/aot/flags.h | 2 +- tensorflow/compiler/aot/tfcompile.bzl | 3 +++ 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc index eac8da0ab1..77c4ec88cb 100644 --- a/tensorflow/compiler/aot/compile.cc +++ b/tensorflow/compiler/aot/compile.cc @@ -97,11 +97,11 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config, TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client, &computation, &compile_result->has_context_arg)); - if (!flags.debug_dir.empty()) { + if (!flags.out_session_module.empty()) { TF_ASSIGN_OR_RETURN(std::unique_ptr module, computation.Snapshot()); - string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb"); - TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module)); + TF_RETURN_IF_ERROR( + WriteBinaryProto(Env::Default(), flags.out_session_module, *module)); } xla::cpu::CpuAotCompilationOptions aot_opts( flags.target_triple, flags.target_cpu, flags.target_features, diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc index 5aff10346f..7c2f27e550 100644 --- a/tensorflow/compiler/aot/flags.cc +++ b/tensorflow/compiler/aot/flags.cc @@ -33,9 +33,6 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) { "fetch nodes will be dumped to stdout in a comma-separated list. " "Typically used to format arguments for other tools, e.g. " "freeze_graph."}, - {"debug_dir", &flags->debug_dir, - "Specifies a directory to dump debugging information, including " - "rewritten graphs and the XLA HLO module."}, // Flags controlling the XLA ahead-of-time compilation, that correspond to // the fields of xla::cpu::CpuAotCompilationOptions. // @@ -64,6 +61,8 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) { "namespaces are given, within the global namespace."}, {"out_object", &flags->out_object, "Output object file name."}, {"out_header", &flags->out_header, "Output header file name."}, + {"out_session_module", &flags->out_session_module, + "Output session module proto."}, {"gen_name_to_index", &flags->gen_name_to_index, "Generate name-to-index data for Lookup{Arg,Result}Index methods."}, {"gen_program_shape", &flags->gen_program_shape, diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h index 3246dbf95c..3519659e3a 100644 --- a/tensorflow/compiler/aot/flags.h +++ b/tensorflow/compiler/aot/flags.h @@ -29,7 +29,6 @@ struct MainFlags { string graph; string config; bool dump_fetch_nodes = false; - string debug_dir; string target_triple; string target_cpu; string target_features; @@ -37,6 +36,7 @@ struct MainFlags { string cpp_class; string out_object; string out_header; + string out_session_module; // C++ codegen options bool gen_name_to_index = false; diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl index 4888760acd..0ecfbedcb4 100644 --- a/tensorflow/compiler/aot/tfcompile.bzl +++ b/tensorflow/compiler/aot/tfcompile.bzl @@ -129,6 +129,7 @@ def tf_library(name, graph, config, # Rule that runs tfcompile to produce the header and object file. header_file = name + ".h" object_file = name + ".o" + session_module_pb = name + "_session_module.pb" ep = ("__" + PACKAGE_NAME + "__" + name).replace("/", "_") native.genrule( name=("gen_" + name), @@ -139,6 +140,7 @@ def tf_library(name, graph, config, outs=[ header_file, object_file, + session_module_pb, ], cmd=("$(location " + tfcompile_tool + ")" + " --graph=$(location " + tfcompile_graph + ")" + @@ -148,6 +150,7 @@ def tf_library(name, graph, config, " --target_triple=" + target_llvm_triple() + " --out_header=$(@D)/" + header_file + " --out_object=$(@D)/" + object_file + + " --out_session_module=$(@D)/" + session_module_pb + " " + (tfcompile_flags or "")), tools=[tfcompile_tool], visibility=visibility, -- GitLab From c0ca50a47724363e5edb4de6afe28f5c60cd2eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sat, 21 Oct 2017 07:08:48 +0800 Subject: [PATCH 228/573] ENH: add Relu6GradGrad (#13268) * ENH: add Relu6GradGrad * TST: add test case * CLN: import nn_grad * TST: add init value --- tensorflow/python/BUILD | 13 ++++++++ tensorflow/python/ops/nn_grad.py | 7 ++++ tensorflow/python/ops/nn_grad_test.py | 48 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 tensorflow/python/ops/nn_grad_test.py diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index b7aa7bbf6b..933e0e3e8c 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -4201,6 +4201,19 @@ cuda_py_test( main = "client/session_benchmark.py", ) +cuda_py_test( + name = "nn_grad_test", + size = "small", + srcs = ["ops/nn_grad_test.py"], + additional_deps = [ + ":client_testlib", + ":framework_for_generated_wrappers", + ":nn_grad", + ":nn_ops", + "//third_party/py/numpy", + ], +) + py_library( name = "tf_item", srcs = [ diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index c7c745142b..557f39fb42 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -352,6 +352,13 @@ def _Relu6Grad(op, grad): return gen_nn_ops._relu6_grad(grad, op.outputs[0]) # pylint: disable=protected-access +@ops.RegisterGradient("Relu6Grad") +def _Relu6GradGrad(op, grad): + x = op.inputs[1] + return (gen_nn_ops._relu6_grad(grad, x), array_ops.zeros( + shape=array_ops.shape(x), dtype=x.dtype)) + + @ops.RegisterGradient("Elu") def _EluGrad(op, grad): return gen_nn_ops._elu_grad(grad, op.outputs[0]) diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py new file mode 100644 index 0000000000..f7541c0e89 --- /dev/null +++ b/tensorflow/python/ops/nn_grad_test.py @@ -0,0 +1,48 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Python ops defined in nn_grad.py.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import nn_grad +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import test + + +class Relu6OpTest(test.TestCase): + def testRelu6GradGrad(self): + inputs = constant_op.constant([[-2, -1, 1, 3], [5, 7, 8, 9]], + dtype=dtypes.float32) + x_init_value = np.array([[-3.5, -1.5, 2, 4], [4.5, 7.5, 8.5, 11]]) + r = nn_ops.relu6(inputs) + r_g = gradients_impl.gradients(r, inputs)[0] + with self.test_session(): + error = gradient_checker.compute_gradient_error( + inputs, inputs.get_shape().as_list(), + r_g, r_g.get_shape().as_list(), + x_init_value=x_init_value) + self.assertLess(error, 1e-4) + + +if __name__ == "__main__": + test.main() -- GitLab From 9c825d32c9423980e1b263a50360e03e833b69a6 Mon Sep 17 00:00:00 2001 From: Jinze Bai Date: Sat, 21 Oct 2017 07:12:31 +0800 Subject: [PATCH 229/573] Merge two GPU kernel launching to one in DiagOp. (#13859) --- tensorflow/core/kernels/diag_op_gpu.cu.cc | 49 +++++++++-------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc index 9878f347d2..684f00ea61 100644 --- a/tensorflow/core/kernels/diag_op_gpu.cu.cc +++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc @@ -33,15 +33,12 @@ __global__ void DiagCudaKernel(const int num_threads, const T* in, T* out) { CUDA_1D_KERNEL_LOOP(index, num_threads) { - out[(1 + size) * index] = in[index]; - } -} - -template -__global__ void ZeroCudaKernel(const int num_threads, - T* out) { - CUDA_1D_KERNEL_LOOP(index, num_threads) { - out[index] = T(0); + // Fill the diagonal elements or set to zero in other place. + if (index % (1 + size) == 0) { + out[index] = in[index / (1 + size)]; + } else { + out[index] = T(0); + } } } @@ -50,39 +47,30 @@ struct DiagFunctor { EIGEN_ALWAYS_INLINE Status operator() (OpKernelContext* context, const int64 size, const T* in, T* out) { - // CudaLaunchConfig uses an int for virtual_thread_count, - // so this may overflow in extreme cases. - if (size && (size * size / size) != size) { - return errors::Internal( - "DiagOp got input size too large."); - } - // Empty tensor couldn't launch the kernel. if (size == 0) { return Status::OK(); } - const GPUDevice& device = context->eigen_device(); - // Set output memory with zero elements. - CudaLaunchConfig zero_config = GetCudaLaunchConfig(size*size, device); - ZeroCudaKernel<<>>( - zero_config.virtual_thread_count, out); - auto err = cudaGetLastError(); - if (err != cudaSuccess) { + // CudaLaunchConfig uses an int for virtual_thread_count, + // so this may overflow for `size*size` in extreme cases, + // here is checking the multiplication overflow for integer. + if (size && (int(size * size) / size) != size) { return errors::Internal( - "Could not launch DiagOp kernel: ", - cudaGetErrorString(err), "."); + "DiagOp got input size too large."); } + int virtual_thread_count = int(size * size); - // Fill the diagonal elements - CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device); + // Launch the GPU kernel. + const GPUDevice& device = context->eigen_device(); + CudaLaunchConfig diag_config = GetCudaLaunchConfig( + virtual_thread_count, device); DiagCudaKernel<<>>( diag_config.virtual_thread_count, size, in, out); - err = cudaGetLastError(); + + auto err = cudaGetLastError(); if (err != cudaSuccess) { return errors::Internal( "Could not launch DiagOp kernel: ", @@ -127,6 +115,7 @@ struct DiagPartFunctor { diag_config.thread_per_block, 0, device.stream()>>>( diag_config.virtual_thread_count, size, in, out); + auto err = cudaGetLastError(); if (err != cudaSuccess) { return errors::Internal( -- GitLab From 49483793695247f27332c7db0b9740e95a5de3db Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Fri, 20 Oct 2017 16:28:55 -0700 Subject: [PATCH 230/573] Make `tf.contrib.distributions` quadrature family accept a `Tensor` for `quadrature_grid_and_probs` argument. PiperOrigin-RevId: 172950094 --- .../bijectors/masked_autoregressive_test.py | 4 +- .../python/kernel_tests/independent_test.py | 4 +- .../kernel_tests/mixture_same_family_test.py | 6 +- .../kernel_tests/poisson_lognormal_test.py | 28 +++++-- .../kernel_tests/vector_diffeomixture_test.py | 53 +++++++++++--- .../vector_sinh_arcsinh_diag_test.py | 12 +-- .../python/ops/poisson_lognormal.py | 53 +++++++------- .../distributions/python/ops/test_util.py | 34 +++++---- .../python/ops/vector_diffeomixture.py | 46 ++++++------ tensorflow/python/ops/distributions/util.py | 73 ++++++++++++++++++- 10 files changed, 216 insertions(+), 97 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py index 98c09545ac..25a9b6f5fe 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py @@ -111,7 +111,7 @@ class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers, event_shape=[dims], validate_args=True) self.run_test_sample_consistent_log_prob( - sess=sess, + sess_run_fn=sess.run, dist=dist, num_samples=int(1e5), radius=1., @@ -130,7 +130,7 @@ class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers, event_shape=[dims], validate_args=True) self.run_test_sample_consistent_log_prob( - sess=sess, + sess_run_fn=sess.run, dist=dist, num_samples=int(1e5), radius=1., diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py index 7a321db4b2..dcc66e8972 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py @@ -23,7 +23,6 @@ import numpy as np from tensorflow.contrib.distributions.python.ops import independent as independent_lib from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib -from tensorflow.contrib.distributions.python.ops import test_util from tensorflow.python.ops import math_ops from tensorflow.python.ops.distributions import normal as normal_lib from tensorflow.python.platform import test @@ -41,8 +40,7 @@ def try_import(name): # pylint: disable=invalid-name stats = try_import("scipy.stats") -class ProductDistributionTest( - test_util.VectorDistributionTestHelpers, test.TestCase): +class ProductDistributionTest(test.TestCase): def testSampleAndLogProbUnivariate(self): loc = np.float32([-1., 1]) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py index ee4f989dac..ece6bc077d 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py @@ -94,10 +94,10 @@ class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers, loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, gm, radius=1., center=[-1., 1], rtol=0.02) + sess.run, gm, radius=1., center=[-1., 1], rtol=0.02) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, gm, radius=1., center=[1., -1], rtol=0.02) + sess.run, gm, radius=1., center=[1., -1], rtol=0.02) def testLogCdf(self): with self.test_session() as sess: @@ -122,7 +122,7 @@ class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers, mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), components_distribution=mvn_diag_lib.MultivariateNormalDiag( loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) - self.run_test_sample_consistent_mean_covariance(sess, gm) + self.run_test_sample_consistent_mean_covariance(sess.run, gm) def testVarianceConsistentCovariance(self): with self.test_session() as sess: diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py index 3ded4159d8..3c0147b8cf 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py @@ -22,6 +22,8 @@ import numpy as np from tensorflow.contrib.distributions.python.ops import poisson_lognormal from tensorflow.contrib.distributions.python.ops import test_util +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -38,7 +40,7 @@ class PoissonLogNormalQuadratureCompoundTest( np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( - sess, pln, rtol=0.1) + sess.run, pln, rtol=0.1) def testMeanVariance(self): with self.test_session() as sess: @@ -49,7 +51,7 @@ class PoissonLogNormalQuadratureCompoundTest( np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( - sess, pln, rtol=0.02) + sess.run, pln, rtol=0.02) def testSampleProbConsistentBroadcastScalar(self): with self.test_session() as sess: @@ -60,7 +62,7 @@ class PoissonLogNormalQuadratureCompoundTest( np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( - sess, pln, rtol=0.1, atol=0.01) + sess.run, pln, rtol=0.1, atol=0.01) def testMeanVarianceBroadcastScalar(self): with self.test_session() as sess: @@ -71,7 +73,7 @@ class PoissonLogNormalQuadratureCompoundTest( np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( - sess, pln, rtol=0.1, atol=0.01) + sess.run, pln, rtol=0.1, atol=0.01) def testSampleProbConsistentBroadcastBoth(self): with self.test_session() as sess: @@ -82,7 +84,7 @@ class PoissonLogNormalQuadratureCompoundTest( np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( - sess, pln, rtol=0.1, atol=0.08) + sess.run, pln, rtol=0.1, atol=0.08) def testMeanVarianceBroadcastBoth(self): with self.test_session() as sess: @@ -93,7 +95,21 @@ class PoissonLogNormalQuadratureCompoundTest( np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( - sess, pln, rtol=0.1, atol=0.01) + sess.run, pln, rtol=0.1, atol=0.01) + + def testSampleProbConsistentDynamicQuadrature(self): + with self.test_session() as sess: + qgrid = array_ops.placeholder(dtype=dtypes.float32) + qprobs = array_ops.placeholder(dtype=dtypes.float32) + g, p = np.polynomial.hermite.hermgauss(deg=10) + pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( + loc=-2., + scale=1.1, + quadrature_grid_and_probs=(g, p), + validate_args=True) + self.run_test_sample_consistent_log_prob( + lambda x: sess.run(x, feed_dict={qgrid: g, qprobs: p}), + pln, rtol=0.1) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py index aea4d42503..de4a221f7b 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py @@ -22,6 +22,8 @@ import numpy as np from tensorflow.contrib.distributions.python.ops import test_util from tensorflow.contrib.distributions.python.ops import vector_diffeomixture as vector_diffeomixture_lib +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.ops.distributions import normal as normal_lib from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib @@ -55,10 +57,10 @@ class VectorDiffeomixtureTest( validate_args=True) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=2., center=0., rtol=0.005) + sess.run, vdm, radius=2., center=0., rtol=0.005) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=4., center=2., rtol=0.005) + sess.run, vdm, radius=4., center=2., rtol=0.005) def testSampleProbConsistentBroadcastMixNonStandardBase(self): with self.test_session() as sess: @@ -83,10 +85,10 @@ class VectorDiffeomixtureTest( validate_args=True) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=2., center=1., rtol=0.006) + sess.run, vdm, radius=2., center=1., rtol=0.006) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=4., center=3., rtol=0.009) + sess.run, vdm, radius=4., center=3., rtol=0.009) def testSampleProbConsistentBroadcastMixBatch(self): with self.test_session() as sess: @@ -114,10 +116,10 @@ class VectorDiffeomixtureTest( validate_args=True) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=2., center=0., rtol=0.005) + sess.run, vdm, radius=2., center=0., rtol=0.005) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=4., center=2., rtol=0.005) + sess.run, vdm, radius=4., center=2., rtol=0.005) def testMeanCovarianceNoBatch(self): with self.test_session() as sess: @@ -141,7 +143,7 @@ class VectorDiffeomixtureTest( ], validate_args=True) self.run_test_sample_consistent_mean_covariance( - sess, vdm, rtol=0.02, cov_rtol=0.06) + sess.run, vdm, rtol=0.02, cov_rtol=0.06) def testMeanCovarianceNoBatchUncenteredNonStandardBase(self): with self.test_session() as sess: @@ -165,7 +167,7 @@ class VectorDiffeomixtureTest( ], validate_args=True) self.run_test_sample_consistent_mean_covariance( - sess, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025) + sess.run, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025) def testMeanCovarianceBatch(self): with self.test_session() as sess: @@ -192,7 +194,40 @@ class VectorDiffeomixtureTest( ], validate_args=True) self.run_test_sample_consistent_mean_covariance( - sess, vdm, rtol=0.02, cov_rtol=0.06) + sess.run, vdm, rtol=0.02, cov_rtol=0.06) + + def testSampleProbConsistentDynamicQuadrature(self): + with self.test_session() as sess: + qgrid = array_ops.placeholder(dtype=dtypes.float32) + qprobs = array_ops.placeholder(dtype=dtypes.float32) + g, p = np.polynomial.hermite.hermgauss(deg=8) + dims = 4 + vdm = vector_diffeomixture_lib.VectorDiffeomixture( + mix_loc=[[0.], [1.]], + mix_scale=[1.], + distribution=normal_lib.Normal(0., 1.), + loc=[ + None, + np.float32([2.]*dims), + ], + scale=[ + linop_identity_lib.LinearOperatorScaledIdentity( + num_rows=dims, + multiplier=np.float32(1.1), + is_positive_definite=True), + linop_diag_lib.LinearOperatorDiag( + diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), + is_positive_definite=True), + ], + quadrature_grid_and_probs=(g, p), + validate_args=True) + # Ball centered at component0's mean. + sess_run_fn = lambda x: sess.run(x, feed_dict={qgrid: g, qprobs: p}) + self.run_test_sample_consistent_log_prob( + sess_run_fn, vdm, radius=2., center=0., rtol=0.005) + # Larger ball centered at component1's mean. + self.run_test_sample_consistent_log_prob( + sess_run_fn, vdm, radius=4., center=2., rtol=0.005) # TODO(jvdillon): We've tested that (i) .sample and .log_prob are consistent, # (ii) .mean, .stddev etc... and .sample are consistent. However, we haven't diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py index a5d837d454..2bc6a926dd 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py @@ -210,15 +210,15 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers, validate_args=True) self.run_test_sample_consistent_log_prob( - sess, sasnorm, radius=1.0, center=0., rtol=0.1) + sess.run, sasnorm, radius=1.0, center=0., rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=-0.15, rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=0.15, @@ -237,15 +237,15 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers, validate_args=True) self.run_test_sample_consistent_log_prob( - sess, sasnorm, radius=1.0, center=0., rtol=0.1) + sess.run, sasnorm, radius=1.0, center=0., rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=-0.15, rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=0.15, diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 80d4e2dc5e..8a95038a3c 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np +from tensorflow.contrib.distributions.python.ops import distribution_util from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -29,7 +30,6 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops.distributions import categorical as categorical_lib from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import util as distribution_util __all__ = [ @@ -55,8 +55,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): ``` where `lambda(z) = exp(sqrt(2) scale z + loc)` and the `prob,grid` terms - are from [Gauss--Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). Note that + are from [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). Note that the second line made the substitution: `z(l) = (log(l) - loc) / (sqrt(2) scale)` which implies `lambda(z)` [above] and `dl = sqrt(2) scale lambda(z) dz` @@ -65,8 +67,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): Poisson rate parameter. Unfortunately, the non-approximate distribution lacks an analytical probability density function (pdf). Therefore the `PoissonLogNormalQuadratureCompound` class implements an approximation based - on [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). + on [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). + Note: although the `PoissonLogNormalQuadratureCompound` is approximately the Poisson-LogNormal compound distribution, it is itself a valid distribution. Viz., it possesses a `sample`, `log_prob`, `mean`, `variance`, etc. which are @@ -76,9 +81,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): The `PoissonLogNormalQuadratureCompound` approximates a Poisson-LogNormal [compound distribution]( - https://en.wikipedia.org/wiki/Compound_probability_distribution). - Using variable-substitution and [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature) we can + https://en.wikipedia.org/wiki/Compound_probability_distribution). Using + variable-substitution and [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)) we can redefine the distribution to be a parameter-less convex combination of `deg` different Poisson samples. @@ -125,9 +132,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): the LogNormal prior. scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of the LogNormal prior. - quadrature_grid_and_probs: Python pair of `list`-like objects representing - the sample points and the corresponding (possibly normalized) weight. - When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`. + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. When `None`, defaults to: + `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -140,8 +148,6 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): Raises: TypeError: if `loc.dtype != scale[0].dtype`. - ValueError: if `quadrature_grid_and_probs is not None` and - `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` """ parameters = locals() with ops.name_scope(name, values=[loc, scale]): @@ -157,21 +163,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format( loc.dtype.name, scale.dtype.name)) - if quadrature_grid_and_probs is None: - grid, probs = np.polynomial.hermite.hermgauss(deg=8) - else: - grid, probs = tuple(quadrature_grid_and_probs) - if len(grid) != len(probs): - raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " - "same-length list-like objects") - grid = grid.astype(dtype.as_numpy_dtype) - probs = probs.astype(dtype.as_numpy_dtype) - probs /= np.linalg.norm(probs, ord=1) + grid, probs = distribution_util.process_quadrature_grid_and_probs( + quadrature_grid_and_probs, dtype, validate_args) self._quadrature_grid = grid self._quadrature_probs = probs + self._quadrature_size = distribution_util.dimension_size(probs, axis=0) self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(probs), + logits=math_ops.log(self._quadrature_probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -254,10 +253,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): [batch_size])), seed=distribution_util.gen_new_seed( seed, "poisson_lognormal_quadrature_compound")) - # Stride `quadrature_degree` for `batch_size` number of times. + # Stride `quadrature_size` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * len(self.quadrature_probs), - delta=len(self.quadrature_probs), + limit=batch_size * self._quadrature_size, + delta=self._quadrature_size, dtype=ids.dtype) ids += offset rate = array_ops.gather( diff --git a/tensorflow/contrib/distributions/python/ops/test_util.py b/tensorflow/contrib/distributions/python/ops/test_util.py index 631ffc1bac..77f2a39273 100644 --- a/tensorflow/contrib/distributions/python/ops/test_util.py +++ b/tensorflow/contrib/distributions/python/ops/test_util.py @@ -38,7 +38,7 @@ class DiscreteScalarDistributionTestHelpers(object): """DiscreteScalarDistributionTestHelpers.""" def run_test_sample_consistent_log_prob( - self, sess, dist, + self, sess_run_fn, dist, num_samples=int(1e5), num_threshold=int(1e3), seed=42, rtol=1e-2, atol=0.): """Tests that sample/log_prob are consistent with each other. @@ -51,7 +51,9 @@ class DiscreteScalarDistributionTestHelpers(object): are consistent. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. num_samples: Python `int` scalar indicating the number of Monte-Carlo @@ -87,7 +89,7 @@ class DiscreteScalarDistributionTestHelpers(object): probs = math_ops.exp(dist.log_prob(edges)) probs = array_ops.reshape(probs, shape=[-1, batch_size])[:, b] - [counts_, probs_] = sess.run([counts, probs]) + [counts_, probs_] = sess_run_fn([counts, probs]) valid = counts_ > num_threshold probs_ = probs_[valid] counts_ = counts_[valid] @@ -95,7 +97,7 @@ class DiscreteScalarDistributionTestHelpers(object): rtol=rtol, atol=atol) def run_test_sample_consistent_mean_variance( - self, sess, dist, + self, sess_run_fn, dist, num_samples=int(1e5), seed=24, rtol=1e-2, atol=0.): """Tests that sample/mean/variance are consistent with each other. @@ -104,7 +106,9 @@ class DiscreteScalarDistributionTestHelpers(object): to the same distribution. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. num_samples: Python `int` scalar indicating the number of Monte-Carlo @@ -130,7 +134,7 @@ class DiscreteScalarDistributionTestHelpers(object): mean_, variance_, stddev_ - ] = sess.run([ + ] = sess_run_fn([ sample_mean, sample_variance, sample_stddev, @@ -187,7 +191,7 @@ class VectorDistributionTestHelpers(object): def run_test_sample_consistent_log_prob( self, - sess, + sess_run_fn, dist, num_samples=int(1e5), radius=1., @@ -240,7 +244,9 @@ class VectorDistributionTestHelpers(object): https://en.wikipedia.org/wiki/Importance_sampling. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. The distribution must have non-zero probability of sampling every point @@ -301,8 +307,8 @@ class VectorDistributionTestHelpers(object): init_op = variables_ops.global_variables_initializer() # Execute graph. - sess.run(init_op) - [batch_shape_, actual_volume_, sample_volume_] = sess.run([ + sess_run_fn(init_op) + [batch_shape_, actual_volume_, sample_volume_] = sess_run_fn([ batch_shape, actual_volume, sample_volume]) # Check results. @@ -312,7 +318,7 @@ class VectorDistributionTestHelpers(object): def run_test_sample_consistent_mean_covariance( self, - sess, + sess_run_fn, dist, num_samples=int(1e5), seed=24, @@ -326,7 +332,9 @@ class VectorDistributionTestHelpers(object): to the same distribution. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. num_samples: Python `int` scalar indicating the number of Monte-Carlo @@ -360,7 +368,7 @@ class VectorDistributionTestHelpers(object): covariance_, variance_, stddev_ - ] = sess.run([ + ] = sess_run_fn([ sample_mean, sample_covariance, sample_variance, diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py index 33dad811a9..92043d6a08 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py @@ -73,8 +73,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): denotes matrix multiplication. However, the non-approximate distribution does not have an analytical probability density function (pdf). Therefore the `VectorDiffeomixture` class implements an approximation based on - [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). I.e., in + [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). I.e., in Note: although the `VectorDiffeomixture` is approximately the `SoftmaxNormal-Distribution` compound distribution, it is itself a valid distribution. It possesses a `sample`, `log_prob`, `mean`, `covariance` which @@ -109,8 +111,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): The `VectorDiffeomixture` approximates a SoftmaxNormal-mixed ("prior") [compound distribution]( https://en.wikipedia.org/wiki/Compound_probability_distribution). - Using variable-substitution and [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature) we can + Using variable-substitution and [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)) we can redefine the distribution to be a parameter-less convex combination of `K` different affine combinations of a `d` iid samples from `distribution`. @@ -141,7 +145,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): and, ```none - grid, weight = np.polynomial.hermite.hermgauss(quadrature_degree) + grid, weight = np.polynomial.hermite.hermgauss(quadrature_size) prob[k] = weight[k] / sqrt(pi) lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i]) ``` @@ -248,9 +252,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): `k`-th element represents the `scale` used for the `k`-th affine transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`, `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices - quadrature_grid_and_probs: Python pair of `list`-like objects representing - the sample points and the corresponding (possibly normalized) weight. - When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`. + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. When `None`, defaults to: + `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -317,24 +322,17 @@ class VectorDiffeomixture(distribution_lib.Distribution): raise NotImplementedError("Currently only bimixtures are supported; " "len(scale)={} is not 2.".format(len(scale))) - if quadrature_grid_and_probs is None: - grid, probs = np.polynomial.hermite.hermgauss(deg=8) - else: - grid, probs = tuple(quadrature_grid_and_probs) - if len(grid) != len(probs): - raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " - "same-length list-like objects") - grid = grid.astype(dtype.as_numpy_dtype) - probs = probs.astype(dtype.as_numpy_dtype) - probs /= np.linalg.norm(probs, ord=1) + grid, probs = distribution_util.process_quadrature_grid_and_probs( + quadrature_grid_and_probs, dtype, validate_args) self._quadrature_grid = grid self._quadrature_probs = probs + self._quadrature_size = distribution_util.dimension_size(probs, axis=0) # Note: by creating the logits as `log(prob)` we ensure that # `self.mixture_distribution.logits` is equivalent to # `math_ops.log(self.mixture_distribution.probs)`. self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(probs), + logits=math_ops.log(probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -361,10 +359,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): validate_args=validate_args, name="interpolated_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip( - interpolate_loc(len(self._quadrature_grid), + interpolate_loc(self._quadrature_size, self._interpolate_weight, loc), - interpolate_scale(len(self._quadrature_grid), + interpolate_scale(self._quadrature_size, self._interpolate_weight, scale)))] @@ -463,10 +461,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): seed=distribution_util.gen_new_seed( seed, "vector_diffeomixture")) - # Stride `quadrature_degree` for `batch_size` number of times. + # Stride `quadrature_size` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * len(self.quadrature_probs), - delta=len(self.quadrature_probs), + limit=batch_size * self._quadrature_size, + delta=self._quadrature_size, dtype=ids.dtype) weight = array_ops.gather( diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py index f261d996b5..41b86f7940 100644 --- a/tensorflow/python/ops/distributions/util.py +++ b/tensorflow/python/ops/distributions/util.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn @@ -1049,13 +1050,77 @@ def dimension_size(x, axis): """Returns the size of a specific dimension.""" # Since tf.gather isn't "constant-in, constant-out", we must first check the # static shape or fallback to dynamic shape. - num_rows = (None if x.get_shape().ndims is None - else x.get_shape()[axis].value) - if num_rows is not None: - return num_rows + s = x.shape.with_rank_at_least(axis + 1)[axis].value + if axis > -1 and s is not None: + return s return array_ops.shape(x)[axis] +def process_quadrature_grid_and_probs( + quadrature_grid_and_probs, dtype, validate_args, name=None): + """Validates quadrature grid, probs or computes them as necessary. + + Args: + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. When `None`, defaults to: + `np.polynomial.hermite.hermgauss(deg=8)`. + dtype: The expected `dtype` of `grid` and `probs`. + validate_args: Python `bool`, default `False`. When `True` distribution + parameters are checked for validity despite possibly degrading runtime + performance. When `False` invalid inputs may silently render incorrect + outputs. + name: Python `str` name prefixed to Ops created by this class. + + Returns: + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. + + Raises: + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` + """ + with ops.name_scope(name, "process_quadrature_grid_and_probs", + [quadrature_grid_and_probs]): + if quadrature_grid_and_probs is None: + grid, probs = np.polynomial.hermite.hermgauss(deg=8) + grid = grid.astype(dtype.as_numpy_dtype) + probs = probs.astype(dtype.as_numpy_dtype) + probs /= np.linalg.norm(probs, ord=1, keepdims=True) + grid = ops.convert_to_tensor(grid, name="grid", dtype=dtype) + probs = ops.convert_to_tensor(probs, name="probs", dtype=dtype) + return grid, probs + + grid, probs = tuple(quadrature_grid_and_probs) + grid = ops.convert_to_tensor(grid, name="grid", dtype=dtype) + probs = ops.convert_to_tensor(probs, name="unnormalized_probs", + dtype=dtype) + probs /= linalg_ops.norm(probs, ord=1, axis=-1, keep_dims=True, + name="probs") + + def _static_dim_size(x, axis): + """Returns the static size of a specific dimension or `None`.""" + return x.shape.with_rank_at_least(axis + 1)[axis].value + + m, n = _static_dim_size(probs, axis=0), _static_dim_size(grid, axis=0) + if m is not None and n is not None: + if m != n: + raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length zero-th-dimension `Tensor`s " + "(saw lengths {}, {})".format(m, n)) + elif validate_args: + grid = control_flow_ops.with_dependencies([ + check_ops.assert_equal( + dimension_size(probs, axis=0), + dimension_size(grid, axis=0), + message=("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length zero-th-dimension `Tensor`s")), + ], grid) + + return grid, probs + + class AppendDocstring(object): """Helper class to promote private subclass docstring to public counterpart. -- GitLab From c77090a0ae61fc69fcdff7c58be9feb6121e3bd4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Oct 2017 16:36:31 -0700 Subject: [PATCH 231/573] Fix issues where int64 crops could not be passed to batch_to_space. (#13862) * Fix issues where int64 crops could not be passed to batch_to_space. This fix tries to address the issue where int64 `crops` could not be passed to `batch_to_space` even though both int32 and int64 are specified as supported in the docs (tf.batch_to_space.__doc__) The reason is that BatchToSpace kernel puts a constraint of int32 to crops data types. This fix removed the constraint so that int64 `crops` could be supported. NOTE: Just removing the constraint should work and it is not necessary to add specification to the kernel class template, as `SubtleMustCopyFlat` called in the class already correctly handled both int32 and int64 cases. Besides, other data types (e.g., float or double) will not be passed to the kernel as they are guarded by the specification in `array_ops.cc`. Signed-off-by: Yong Tang * Also remove int64/int32 type constraints for SpaceToBatch kernels Signed-off-by: Yong Tang * Add test cases for int64 crops of batch_to_space and space_to_batch Signed-off-by: Yong Tang * Fix test failures. Signed-off-by: Yong Tang --- tensorflow/core/kernels/batchtospace_op.cc | 50 ++++++++----------- tensorflow/core/kernels/spacetobatch_op.cc | 50 ++++++++----------- .../kernel_tests/batchtospace_op_test.py | 36 +++++++------ 3 files changed, 65 insertions(+), 71 deletions(-) diff --git a/tensorflow/core/kernels/batchtospace_op.cc b/tensorflow/core/kernels/batchtospace_op.cc index 99b5d3daaa..c1c0d6d329 100644 --- a/tensorflow/core/kernels/batchtospace_op.cc +++ b/tensorflow/core/kernels/batchtospace_op.cc @@ -249,40 +249,34 @@ class BatchToSpaceOp : public OpKernel { Tensor block_shape_; }; -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tcrops") \ - .HostMemory("block_shape") \ - .HostMemory("crops"), \ - BatchToSpaceNDOp); \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("crops"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("crops"), \ + BatchToSpaceNDOp); \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("crops"), \ BatchToSpaceOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER); #undef REGISTER #if GOOGLE_CUDA -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tcrops") \ - .HostMemory("block_shape") \ - .HostMemory("crops"), \ - BatchToSpaceNDOp); \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("crops"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("crops"), \ + BatchToSpaceNDOp); \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("crops"), \ BatchToSpaceOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER); diff --git a/tensorflow/core/kernels/spacetobatch_op.cc b/tensorflow/core/kernels/spacetobatch_op.cc index c513683918..95c1f5e7e8 100644 --- a/tensorflow/core/kernels/spacetobatch_op.cc +++ b/tensorflow/core/kernels/spacetobatch_op.cc @@ -248,40 +248,34 @@ class SpaceToBatchOp : public OpKernel { Tensor block_shape_; }; -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("block_shape") \ - .HostMemory("paddings"), \ - SpaceToBatchNDOp); \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("paddings"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("paddings"), \ + SpaceToBatchNDOp); \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("paddings"), \ SpaceToBatchOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER); #undef REGISTER #if GOOGLE_CUDA -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("block_shape") \ - .HostMemory("paddings"), \ - SpaceToBatchNDOp); \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("paddings"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("paddings"), \ + SpaceToBatchNDOp); \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("paddings"), \ SpaceToBatchOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER); diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py index 8ec93119f2..0c802476a0 100644 --- a/tensorflow/python/kernel_tests/batchtospace_op_test.py +++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py @@ -24,6 +24,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -52,14 +53,15 @@ class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl): def testDepthToSpaceTranspose(self): x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7]) block_size = 2 - crops = np.zeros((2, 2), dtype=np.int32) - y1 = self.batch_to_space(x, crops, block_size=block_size) - y2 = array_ops.transpose( - array_ops.depth_to_space( - array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size), - [3, 1, 2, 0]) - with self.test_session(): - self.assertAllEqual(y1.eval(), y2.eval()) + for crops_dtype in [dtypes.int64, dtypes.int32]: + crops = array_ops.zeros((2, 2), dtype=crops_dtype) + y1 = self.batch_to_space(x, crops, block_size=block_size) + y2 = array_ops.transpose( + array_ops.depth_to_space( + array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size), + [3, 1, 2, 0]) + with self.test_session(): + self.assertAllEqual(y1.eval(), y2.eval()) class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl): @@ -287,9 +289,10 @@ class BatchToSpaceGradientCppTest(BatchToSpaceGradientTest, CppOpImpl): class BatchToSpaceNDGradientTest(test.TestCase): # Check the gradients. - def _checkGrad(self, x, block_shape, crops): + def _checkGrad(self, x, block_shape, crops, crops_dtype): block_shape = np.array(block_shape) - crops = np.array(crops).reshape((len(block_shape), 2)) + crops = constant_op.constant( + np.array(crops).reshape((len(block_shape), 2)), crops_dtype) with self.test_session(): tf_x = ops.convert_to_tensor(x) tf_y = array_ops.batch_to_space_nd(tf_x, block_shape, crops) @@ -304,23 +307,26 @@ class BatchToSpaceNDGradientTest(test.TestCase): self.assertAllClose(x_jacob_t, x_jacob_n, rtol=1e-2, atol=epsilon) - def _compare(self, input_shape, block_shape, crops): + def _compare(self, input_shape, block_shape, crops, crops_dtype): input_shape = list(input_shape) input_shape[0] *= np.prod(block_shape) x = np.random.normal( 0, 1, np.prod(input_shape)).astype(np.float32).reshape(input_shape) - self._checkGrad(x, block_shape, crops) + self._checkGrad(x, block_shape, crops, crops_dtype) # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. def testSmall(self): - self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype) def testSmall2(self): - self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype) def testSmallCrop1x1(self): - self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype) if __name__ == "__main__": -- GitLab From a5fe66b1519668505c0daf5f2d93a4d532cedda1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 16:32:24 -0700 Subject: [PATCH 232/573] Removed some unnecessary broadcasts in binary ops where only one input needs broadcasting (which is a fairly common case, even in the fallback path). PiperOrigin-RevId: 172950493 --- tensorflow/core/kernels/cwise_ops_common.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h index 9a05e1500f..2454620776 100644 --- a/tensorflow/core/kernels/cwise_ops_common.h +++ b/tensorflow/core/kernels/cwise_ops_common.h @@ -410,10 +410,20 @@ struct BinaryFunctor { } } - // Fallback path. Always work and probably slower. - auto lhs = in0.broadcast(bcast0); - auto rhs = in1.broadcast(bcast1); - Assign(dev, out, lhs.binaryExpr(rhs, func)); + // Fallback path. Always works and probably slower. + if (AllOne(bcast0) && AllOne(bcast1)) { + Assign(dev, out, in0.binaryExpr(in1, func)); + } else if (AllOne(bcast0)) { + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, in0.binaryExpr(rhs, func)); + } else if (AllOne(bcast1)) { + auto lhs = in0.broadcast(bcast0); + Assign(dev, out, lhs.binaryExpr(in1, func)); + } else { + auto lhs = in0.broadcast(bcast0); + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, lhs.binaryExpr(rhs, func)); + } } }; -- GitLab From f758b24a825e9787bd0bc89c5e2869116e5384fe Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 20 Oct 2017 16:57:13 -0700 Subject: [PATCH 233/573] Variable name for the eager test (#13873) --- .../contrib/framework/python/ops/accumulate_n_v2_eager_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py index f3453f89fa..5f086ea8cc 100644 --- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py +++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py @@ -64,7 +64,8 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase): np.random.seed(42) num_inputs = 3 input_vars = [ - resource_variable_ops.ResourceVariable(10.0 * np.random.random()) + resource_variable_ops.ResourceVariable(10.0 * np.random.random(), + name="t1") for i in range(0, num_inputs) ] -- GitLab From 29c7b46585aabab6b1a1677324667c2d5720181c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 17:26:46 -0700 Subject: [PATCH 234/573] Adding the Stanford Tensorflow class to community resources. PiperOrigin-RevId: 172956049 --- tensorflow/docs_src/community/welcome.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md index 4991783a53..c4f78051f0 100644 --- a/tensorflow/docs_src/community/welcome.md +++ b/tensorflow/docs_src/community/welcome.md @@ -29,6 +29,7 @@ The TensorFlow community has created many great projects around TensorFlow, incl * [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow) * [Edward - A library for probabilistic modeling, inference, and criticism](http://edwardlib.org) ([Github](https://github.com/blei-lab/edward), [Forum](https://discourse.edwardlib.org)) * [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow) +* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - Please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource. ## TensorFlow Communities Around the World -- GitLab From df8bce63d6de6e728e69eb9f45862b816f88a0db Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 20 Oct 2017 17:40:40 -0700 Subject: [PATCH 235/573] Fix crash when `int64` axis is passed to `tf.reduce_sum` (#13863) * Fix crash when `int64` axis is passed to `tf.reduce_sum` This fix tries to fix the crash triggered by `int64` axis passed to `tf.reduce_sum`: ``` ubuntu@ubuntu:~/tensorflow2$ (cd && python) Python 2.7.12 (default, Nov 19 2016, 06:48:10) [GCC 5.4.0 20160609] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import tensorflow as tf >>> v = tf.reduce_sum([1,2,3], tf.constant(0, tf.int64)) 2017-10-20 15:55:06.993430: F tensorflow/core/framework/tensor.cc:601] Check failed: dtype() == expected_dtype (9 vs. 3) ubuntu@ubuntu:~/tensorflow2$ ``` The issue is caused by the fact that shape inference in `common_shape_fns.cc` only assumes int32 without proper handling of diffent types. In `math_ops.cc` both int32 and int64 are mentioned. NOTE that this fix does not address the issue that int64 is not supported. To allow int64 axis it is more than adding a template in `ReductionOp` as the type of the axis seems to be decided by some other ways in Eigen. This fix merely fixed the crash so that an error message will return without exit from the python program "No OpKernel was registered to support Op 'Sum' with these attrs". Still, I think its worth to at least allow the program to continue in case of unsupported kernel. Signed-off-by: Yong Tang * Update implementation with a template helper function. Signed-off-by: Yong Tang --- tensorflow/core/framework/common_shape_fns.cc | 58 ++++++++++++------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index 4796c3c00a..315c99d32b 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -1020,6 +1020,29 @@ Status UnknownShape(shape_inference::InferenceContext* c) { return Status::OK(); } +template +Status ReductionShapeHelper(const Tensor* reduction_indices_t, + const int32 input_rank, + std::set& true_indices) { + auto reduction_indices = reduction_indices_t->flat(); + for (int i = 0; i < reduction_indices_t->NumElements(); ++i) { + const T reduction_index = reduction_indices(i); + if (reduction_index < -input_rank || reduction_index >= input_rank) { + return errors::InvalidArgument("Invalid reduction dimension ", + reduction_index, " for input with ", + input_rank, " dimensions."); + } + + auto wrapped_index = reduction_index; + if (wrapped_index < 0) { + wrapped_index += input_rank; + } + + true_indices.insert(wrapped_index); + } + return Status::OK(); +} + Status ReductionShape(InferenceContext* c) { ShapeHandle input = c->input(0); @@ -1050,22 +1073,16 @@ Status ReductionShape(InferenceContext* c) { } const int32 input_rank = c->Rank(input); - std::set true_indices; - auto reduction_indices = reduction_indices_t->flat(); - for (int i = 0; i < reduction_indices_t->NumElements(); ++i) { - int32 reduction_index = reduction_indices(i); - if (reduction_index < -input_rank || reduction_index >= input_rank) { - return errors::InvalidArgument("Invalid reduction dimension ", - reduction_index, " for input with ", - input_rank, " dimensions."); - } - - int32 wrapped_index = reduction_index; - if (wrapped_index < 0) { - wrapped_index += input_rank; - } - - true_indices.insert(wrapped_index); + std::set true_indices; + if (reduction_indices_t->dtype() == DataType::DT_INT32) { + TF_RETURN_IF_ERROR(ReductionShapeHelper(reduction_indices_t, + input_rank, true_indices)); + } else if (reduction_indices_t->dtype() == DataType::DT_INT64) { + TF_RETURN_IF_ERROR(ReductionShapeHelper(reduction_indices_t, + input_rank, true_indices)); + } else { + return errors::InvalidArgument( + "reduction_indices can only be int32 or int64"); } std::vector dims; @@ -1319,11 +1336,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) { Status s = c->Merge(prefix_indices, prefix_updates, &unused); if (!s.ok()) { return errors::InvalidArgument( - "The outer ", num_outer_dims, - " dimensions of indices.shape=", c->DebugString(indices_shape), - " must match the outer ", num_outer_dims, - " dimensions of updates.shape=", c->DebugString(updates_shape), - ": ", s.error_message()); + "The outer ", num_outer_dims, " dimensions of indices.shape=", + c->DebugString(indices_shape), " must match the outer ", + num_outer_dims, " dimensions of updates.shape=", + c->DebugString(updates_shape), ": ", s.error_message()); } ShapeHandle input_suffix; -- GitLab From d7409d32bba5ffa89141ec5427780f68a3b6942d Mon Sep 17 00:00:00 2001 From: Simone Cirillo Date: Sat, 21 Oct 2017 02:44:08 +0200 Subject: [PATCH 236/573] Fix import of spatial_softmax from tensorflow.contrib.layers (#13833) --- tensorflow/contrib/layers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py index d8ab7c2d70..d309ba958d 100644 --- a/tensorflow/contrib/layers/__init__.py +++ b/tensorflow/contrib/layers/__init__.py @@ -47,6 +47,7 @@ See the @{$python/contrib.layers} guide. @@separable_conv2d @@separable_convolution2d @@softmax +@@spatial_softmax @@stack @@unit_norm @@bow_encoder -- GitLab From 62df65c7255e2a8878cd29f66fe80ff8952de157 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Oct 2017 17:46:17 -0700 Subject: [PATCH 237/573] Add dtype argument to Mean and Accuracy object-oriented metrics. PiperOrigin-RevId: 172957714 --- .../contrib/eager/python/metrics_impl.py | 27 +++++++++++-------- .../contrib/eager/python/metrics_test.py | 20 ++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 2a624b218c..2139c2b4b9 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -198,13 +198,19 @@ class Mean(Metric): # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64? # Or defaults to type of the input if it is tf.float32, else tf.float64? - def build(self, values, weights=None): - del values, weights # build() does not use call's arguments + def __init__(self, name=None, dtype=dtypes.float64): + super(Mean, self).__init__(name=name) + self.dtype = dtype + + def build(self, *args, **kwargs): + # build() does not use call's arguments, by using *args, **kwargs + # we make it easier to inherit from Mean(). + del args, kwargs self.numer = self.add_variable(name="numer", shape=(), - dtype=dtypes.float64, + dtype=self.dtype, initializer=init_ops.zeros_initializer) self.denom = self.add_variable(name="denom", shape=(), - dtype=dtypes.float64, + dtype=self.dtype, initializer=init_ops.zeros_initializer) def call(self, values, weights=None): @@ -219,13 +225,13 @@ class Mean(Metric): """ if weights is None: self.denom.assign_add( - math_ops.cast(array_ops.size(values), dtypes.float64)) + math_ops.cast(array_ops.size(values), self.dtype)) values = math_ops.reduce_sum(values) - self.numer.assign_add(math_ops.cast(values, dtypes.float64)) + self.numer.assign_add(math_ops.cast(values, self.dtype)) else: - weights = math_ops.cast(weights, dtypes.float64) + weights = math_ops.cast(weights, self.dtype) self.denom.assign_add(math_ops.reduce_sum(weights)) - values = math_ops.cast(values, dtypes.float64) * weights + values = math_ops.cast(values, self.dtype) * weights self.numer.assign_add(math_ops.reduce_sum(values)) def result(self): @@ -235,9 +241,8 @@ class Mean(Metric): class Accuracy(Mean): """Calculates how often `predictions` matches `labels`.""" - def build(self, labels, predictions, weights=None): - del labels, predictions, weights - super(Accuracy, self).build(None) # Arguments are unused + def __init__(self, name=None, dtype=dtypes.float64): + super(Accuracy, self).__init__(name=name, dtype=dtype) def call(self, labels, predictions, weights=None): """Accumulate accuracy statistics. diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index bfb79cd72e..9743666c89 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -34,6 +34,8 @@ class MetricsTest(test.TestCase): m(1000) m([10000.0, 100000.0]) self.assertEqual(111111.0/6, m.result().numpy()) + self.assertEqual(dtypes.float64, m.dtype) + self.assertEqual(dtypes.float64, m.result().dtype) def testWeightedMean(self): m = metrics.Mean() @@ -41,6 +43,14 @@ class MetricsTest(test.TestCase): m([500000, 5000, 500]) # weights of 1 each self.assertNear(535521/4.5, m.result().numpy(), 0.001) + def testMeanDtype(self): + # Can override default dtype of float64. + m = metrics.Mean(dtype=dtypes.float32) + m([0, 2]) + self.assertEqual(1, m.result().numpy()) + self.assertEqual(dtypes.float32, m.dtype) + self.assertEqual(dtypes.float32, m.result().dtype) + def testAccuracy(self): m = metrics.Accuracy() m([0, 1, 2, 3], [0, 0, 0, 0]) # 1 correct @@ -49,6 +59,8 @@ class MetricsTest(test.TestCase): m([6], [6]) # 1 correct m([7], [2]) # 0 correct self.assertEqual(3.0/8, m.result().numpy()) + self.assertEqual(dtypes.float64, m.dtype) + self.assertEqual(dtypes.float64, m.result().dtype) def testWeightedAccuracy(self): m = metrics.Accuracy() @@ -60,6 +72,14 @@ class MetricsTest(test.TestCase): m([7], [2]) # 0 correct, weight 1 self.assertEqual(2.5/5, m.result().numpy()) + def testAccuracyDtype(self): + # Can override default dtype of float64. + m = metrics.Accuracy(dtype=dtypes.float32) + m([0, 0], [0, 1]) + self.assertEqual(0.5, m.result().numpy()) + self.assertEqual(dtypes.float32, m.dtype) + self.assertEqual(dtypes.float32, m.result().dtype) + def testTwoMeans(self): # Verify two metrics with the same class and name don't # accidentally share state. -- GitLab From 0d6a2e35312c71cf8a145a7c40d69883e254daee Mon Sep 17 00:00:00 2001 From: Anna R Date: Fri, 20 Oct 2017 18:19:13 -0700 Subject: [PATCH 238/573] Internal change. PiperOrigin-RevId: 172960439 --- tensorflow/contrib/makefile/proto_text_pb_cc_files.txt | 1 + tensorflow/contrib/makefile/proto_text_pb_h_files.txt | 1 + tensorflow/contrib/makefile/tf_pb_text_files.txt | 1 + tensorflow/contrib/makefile/tf_proto_files.txt | 1 + 4 files changed, 4 insertions(+) diff --git a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt index 5ade8942af..938c4a53ab 100644 --- a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt +++ b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt @@ -24,6 +24,7 @@ tensorflow/core/framework/summary.pb.cc tensorflow/core/framework/step_stats.pb.cc tensorflow/core/framework/resource_handle.pb.cc tensorflow/core/framework/remote_fused_graph_execute_info.pb.cc +tensorflow/core/framework/api_def.pb.cc tensorflow/core/framework/op_def.pb.cc tensorflow/core/framework/node_def.pb.cc tensorflow/core/framework/log_memory.pb.cc diff --git a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt index 1f0ad06cdc..aa91b2f954 100644 --- a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt +++ b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt @@ -25,6 +25,7 @@ tensorflow/core/framework/summary.pb.h tensorflow/core/framework/step_stats.pb.h tensorflow/core/framework/resource_handle.pb.h tensorflow/core/framework/remote_fused_graph_execute_info.pb.h +tensorflow/core/framework/api_def.pb.h tensorflow/core/framework/op_def.pb.h tensorflow/core/framework/node_def.pb.h tensorflow/core/framework/log_memory.pb.h diff --git a/tensorflow/contrib/makefile/tf_pb_text_files.txt b/tensorflow/contrib/makefile/tf_pb_text_files.txt index c39257ffa9..b5431df2eb 100644 --- a/tensorflow/contrib/makefile/tf_pb_text_files.txt +++ b/tensorflow/contrib/makefile/tf_pb_text_files.txt @@ -17,6 +17,7 @@ tensorflow/core/framework/summary.pb_text.cc tensorflow/core/framework/step_stats.pb_text.cc tensorflow/core/framework/resource_handle.pb_text.cc tensorflow/core/framework/remote_fused_graph_execute_info.pb_text.cc +tensorflow/core/framework/api_def.pb_text.cc tensorflow/core/framework/op_def.pb_text.cc tensorflow/core/framework/node_def.pb_text.cc tensorflow/core/framework/log_memory.pb_text.cc diff --git a/tensorflow/contrib/makefile/tf_proto_files.txt b/tensorflow/contrib/makefile/tf_proto_files.txt index a1a9aa7190..d569bde637 100644 --- a/tensorflow/contrib/makefile/tf_proto_files.txt +++ b/tensorflow/contrib/makefile/tf_proto_files.txt @@ -30,6 +30,7 @@ tensorflow/core/framework/step_stats.proto tensorflow/core/framework/resource_handle.proto tensorflow/core/framework/remote_fused_graph_execute_info.proto tensorflow/core/framework/reader_base.proto +tensorflow/core/framework/api_def.proto tensorflow/core/framework/op_def.proto tensorflow/core/framework/node_def.proto tensorflow/core/framework/log_memory.proto -- GitLab From 93e8f3c67d82c2d43b8dddd4cb8d7f02259d0e7e Mon Sep 17 00:00:00 2001 From: Anna R Date: Fri, 20 Oct 2017 18:20:05 -0700 Subject: [PATCH 239/573] Adding Python ApiDef overrides. PiperOrigin-RevId: 172960496 --- tensorflow/core/BUILD | 5 + .../core/api_def/python_api/api_def_B.pbtxt | 18 ++ .../core/api_def/python_api/api_def_C.pbtxt | 15 ++ .../core/api_def/python_api/api_def_D.pbtxt | 54 ++++++ .../core/api_def/python_api/api_def_E.pbtxt | 30 +++ .../core/api_def/python_api/api_def_F.pbtxt | 21 +++ .../core/api_def/python_api/api_def_H.pbtxt | 6 + .../core/api_def/python_api/api_def_I.pbtxt | 15 ++ .../core/api_def/python_api/api_def_L.pbtxt | 24 +++ .../core/api_def/python_api/api_def_M.pbtxt | 78 ++++++++ .../core/api_def/python_api/api_def_Q.pbtxt | 27 +++ .../core/api_def/python_api/api_def_R.pbtxt | 36 ++++ .../core/api_def/python_api/api_def_S.pbtxt | 36 ++++ tensorflow/tools/api/tests/BUILD | 15 ++ .../tools/api/tests/api_compatibility_test.py | 177 ++++++++++++++++++ .../tools/api/tests/convert_from_multiline.cc | 63 +++++++ 16 files changed, 620 insertions(+) create mode 100644 tensorflow/core/api_def/python_api/api_def_B.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_C.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_D.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_E.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_F.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_H.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_I.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_L.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_M.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_Q.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_R.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_S.pbtxt create mode 100644 tensorflow/tools/api/tests/convert_from_multiline.cc diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index d198a796a7..6ad93a73f4 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -3326,6 +3326,11 @@ filegroup( data = glob(["api_def/base_api/*"]), ) +filegroup( + name = "python_api_def", + data = glob(["api_def/python_api/*"]), +) + tf_cc_test( name = "api_test", srcs = ["api_def/api_test.cc"], diff --git a/tensorflow/core/api_def/python_api/api_def_B.pbtxt b/tensorflow/core/api_def/python_api/api_def_B.pbtxt new file mode 100644 index 0000000000..9b5df58eba --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_B.pbtxt @@ -0,0 +1,18 @@ +op { + graph_op_name: "BitwiseAnd" + endpoint { + name: "bitwise.bitwise_and" + } +} +op { + graph_op_name: "BitwiseOr" + endpoint { + name: "bitwise.bitwise_or" + } +} +op { + graph_op_name: "BitwiseXor" + endpoint { + name: "bitwise.bitwise_xor" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_C.pbtxt b/tensorflow/core/api_def/python_api/api_def_C.pbtxt new file mode 100644 index 0000000000..cf8d0622be --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_C.pbtxt @@ -0,0 +1,15 @@ +op { + graph_op_name: "Cholesky" + endpoint { + name: "cholesky" + } + endpoint { + name: "linalg.cholesky" + } +} +op { + graph_op_name: "CropAndResize" + endpoint { + name: "image.crop_and_resize" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_D.pbtxt b/tensorflow/core/api_def/python_api/api_def_D.pbtxt new file mode 100644 index 0000000000..12e0dbec1c --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_D.pbtxt @@ -0,0 +1,54 @@ +op { + graph_op_name: "DecodeAndCropJpeg" + endpoint { + name: "image.decode_and_crop_jpeg" + } +} +op { + graph_op_name: "DecodeBmp" + endpoint { + name: "image.decode_bmp" + } +} +op { + graph_op_name: "DecodeGif" + endpoint { + name: "image.decode_gif" + } +} +op { + graph_op_name: "DecodeJpeg" + endpoint { + name: "image.decode_jpeg" + } +} +op { + graph_op_name: "DecodePng" + endpoint { + name: "image.decode_png" + } +} +op { + graph_op_name: "DepthwiseConv2dNative" + endpoint { + name: "nn.depthwise_conv2d_native" + } +} +op { + graph_op_name: "DepthwiseConv2dNativeBackpropFilter" + endpoint { + name: "nn.depthwise_conv2d_native_backprop_filter" + } +} +op { + graph_op_name: "DepthwiseConv2dNativeBackpropInput" + endpoint { + name: "nn.depthwise_conv2d_native_backprop_input" + } +} +op { + graph_op_name: "DrawBoundingBoxes" + endpoint { + name: "image.draw_bounding_boxes" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_E.pbtxt b/tensorflow/core/api_def/python_api/api_def_E.pbtxt new file mode 100644 index 0000000000..f6871f7138 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_E.pbtxt @@ -0,0 +1,30 @@ +op { + graph_op_name: "Elu" + endpoint { + name: "nn.elu" + } +} +op { + graph_op_name: "EncodeJpeg" + endpoint { + name: "image.encode_jpeg" + } +} +op { + graph_op_name: "EncodePng" + endpoint { + name: "image.encode_png" + } +} +op { + graph_op_name: "ExtractGlimpse" + endpoint { + name: "image.extract_glimpse" + } +} +op { + graph_op_name: "ExtractJpegShape" + endpoint { + name: "image.extract_jpeg_shape" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_F.pbtxt b/tensorflow/core/api_def/python_api/api_def_F.pbtxt new file mode 100644 index 0000000000..844a1348a3 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_F.pbtxt @@ -0,0 +1,21 @@ +op { + graph_op_name: "FFT" + endpoint { + name: "fft" + } + endpoint { + name: "spectral.fft" + } +} +op { + graph_op_name: "FractionalAvgPool" + endpoint { + name: "nn.fractional_avg_pool" + } +} +op { + graph_op_name: "FractionalMaxPool" + endpoint { + name: "nn.fractional_max_pool" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_H.pbtxt b/tensorflow/core/api_def/python_api/api_def_H.pbtxt new file mode 100644 index 0000000000..55998189f4 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_H.pbtxt @@ -0,0 +1,6 @@ +op { + graph_op_name: "HSVToRGB" + endpoint { + name: "image.hsv_to_rgb" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_I.pbtxt b/tensorflow/core/api_def/python_api/api_def_I.pbtxt new file mode 100644 index 0000000000..6c794fab0d --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_I.pbtxt @@ -0,0 +1,15 @@ +op { + graph_op_name: "IFFT" + endpoint { + name: "ifft" + } + endpoint { + name: "spectral.ifft" + } +} +op { + graph_op_name: "Invert" + endpoint { + name: "bitwise.invert" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_L.pbtxt b/tensorflow/core/api_def/python_api/api_def_L.pbtxt new file mode 100644 index 0000000000..38ba26a8e8 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_L.pbtxt @@ -0,0 +1,24 @@ +op { + graph_op_name: "L2Loss" + endpoint { + name: "nn.l2_loss" + } +} +op { + graph_op_name: "LRN" + endpoint { + name: "nn.local_response_normalization" + } + endpoint { + name: "nn.lrn" + } +} +op { + graph_op_name: "LinSpace" + endpoint { + name: "lin_space" + } + endpoint { + name: "linspace" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_M.pbtxt b/tensorflow/core/api_def/python_api/api_def_M.pbtxt new file mode 100644 index 0000000000..154071f6bc --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_M.pbtxt @@ -0,0 +1,78 @@ +op { + graph_op_name: "MatrixBandPart" + endpoint { + name: "linalg.band_part" + } + endpoint { + name: "matrix_band_part" + } +} +op { + graph_op_name: "MatrixDeterminant" + endpoint { + name: "linalg.det" + } + endpoint { + name: "matrix_determinant" + } +} +op { + graph_op_name: "MatrixDiag" + endpoint { + name: "linalg.diag" + } + endpoint { + name: "matrix_diag" + } +} +op { + graph_op_name: "MatrixDiagPart" + endpoint { + name: "linalg.diag_part" + } + endpoint { + name: "matrix_diag_part" + } +} +op { + graph_op_name: "MatrixInverse" + endpoint { + name: "linalg.inv" + } + endpoint { + name: "matrix_inverse" + } +} +op { + graph_op_name: "MatrixSetDiag" + endpoint { + name: "linalg.set_diag" + } + endpoint { + name: "matrix_set_diag" + } +} +op { + graph_op_name: "MatrixSolve" + endpoint { + name: "linalg.solve" + } + endpoint { + name: "matrix_solve" + } +} +op { + graph_op_name: "MatrixTriangularSolve" + endpoint { + name: "linalg.triangular_solve" + } + endpoint { + name: "matrix_triangular_solve" + } +} +op { + graph_op_name: "MaxPoolWithArgmax" + endpoint { + name: "nn.max_pool_with_argmax" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_Q.pbtxt b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt new file mode 100644 index 0000000000..cba032880f --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt @@ -0,0 +1,27 @@ +op { + graph_op_name: "Qr" + endpoint { + name: "linalg.qr" + } + endpoint { + name: "qr" + } +} +op { + graph_op_name: "QuantizedAvgPool" + endpoint { + name: "nn.quantized_avg_pool" + } +} +op { + graph_op_name: "QuantizedMaxPool" + endpoint { + name: "nn.quantized_max_pool" + } +} +op { + graph_op_name: "QuantizedReluX" + endpoint { + name: "nn.quantized_relu_x" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_R.pbtxt b/tensorflow/core/api_def/python_api/api_def_R.pbtxt new file mode 100644 index 0000000000..9a57e72be0 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_R.pbtxt @@ -0,0 +1,36 @@ +op { + graph_op_name: "RGBToHSV" + endpoint { + name: "image.rgb_to_hsv" + } +} +op { + graph_op_name: "Relu" + endpoint { + name: "nn.relu" + } +} +op { + graph_op_name: "ResizeArea" + endpoint { + name: "image.resize_area" + } +} +op { + graph_op_name: "ResizeBicubic" + endpoint { + name: "image.resize_bicubic" + } +} +op { + graph_op_name: "ResizeBilinear" + endpoint { + name: "image.resize_bilinear" + } +} +op { + graph_op_name: "ResizeNearestNeighbor" + endpoint { + name: "image.resize_nearest_neighbor" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_S.pbtxt b/tensorflow/core/api_def/python_api/api_def_S.pbtxt new file mode 100644 index 0000000000..9c7a39038e --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_S.pbtxt @@ -0,0 +1,36 @@ +op { + graph_op_name: "SdcaFprint" + endpoint { + name: "train.sdca_fprint" + } +} +op { + graph_op_name: "SdcaOptimizer" + endpoint { + name: "train.sdca_optimizer" + } +} +op { + graph_op_name: "SdcaShrinkL1" + endpoint { + name: "train.sdca_shrink_l1" + } +} +op { + graph_op_name: "Selu" + endpoint { + name: "nn.selu" + } +} +op { + graph_op_name: "Softplus" + endpoint { + name: "nn.softplus" + } +} +op { + graph_op_name: "Softsign" + endpoint { + name: "nn.softsign" + } +} diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD index e99cc0572f..a913e35101 100644 --- a/tensorflow/tools/api/tests/BUILD +++ b/tensorflow/tools/api/tests/BUILD @@ -11,10 +11,15 @@ exports_files([ "API_UPDATE_WARNING.txt", ]) +load("//tensorflow:tensorflow.bzl", "tf_cc_binary") + py_test( name = "api_compatibility_test", srcs = ["api_compatibility_test.py"], data = [ + ":convert_from_multiline", + "//tensorflow/core:base_api_def", + "//tensorflow/core:python_api_def", "//tensorflow/tools/api/golden:api_golden", "//tensorflow/tools/api/tests:API_UPDATE_WARNING.txt", "//tensorflow/tools/api/tests:README.txt", @@ -23,6 +28,7 @@ py_test( deps = [ "//tensorflow:tensorflow_py", "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:lib", "//tensorflow/python:platform", "//tensorflow/tools/api/lib:python_object_to_proto_visitor", @@ -31,6 +37,15 @@ py_test( ], ) +tf_cc_binary( + name = "convert_from_multiline", + srcs = ["convert_from_multiline.cc"], + deps = [ + "//tensorflow/core:lib", + "//tensorflow/core:op_gen_lib", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py index 1ffa8fc26c..f350c12d41 100644 --- a/tensorflow/tools/api/tests/api_compatibility_test.py +++ b/tensorflow/tools/api/tests/api_compatibility_test.py @@ -28,8 +28,11 @@ from __future__ import division from __future__ import print_function import argparse +from collections import defaultdict +from operator import attrgetter import os import re +import subprocess import sys import unittest @@ -37,6 +40,7 @@ import tensorflow as tf from google.protobuf import text_format +from tensorflow.core.framework import api_def_pb2 from tensorflow.python.lib.io import file_io from tensorflow.python.platform import resource_loader from tensorflow.python.platform import test @@ -64,6 +68,11 @@ _API_GOLDEN_FOLDER = 'tensorflow/tools/api/golden' _TEST_README_FILE = 'tensorflow/tools/api/tests/README.txt' _UPDATE_WARNING_FILE = 'tensorflow/tools/api/tests/API_UPDATE_WARNING.txt' +_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' +_CONVERT_FROM_MULTILINE_SCRIPT = 'tensorflow/tools/api/tests/convert_from_multiline' +_BASE_API_DIR = 'tensorflow/core/api_def/base_api' +_PYTHON_API_DIR = 'tensorflow/core/api_def/python_api' + def _KeyToFilePath(key): """From a given key, construct a filepath.""" @@ -88,6 +97,30 @@ def _FileNameToKey(filename): return api_object_key +def _GetSymbol(symbol_id): + """Get TensorFlow symbol based on the given identifier. + + Args: + symbol_id: Symbol identifier in the form module1.module2. ... .sym. + + Returns: + Symbol corresponding to the given id. + """ + # Ignore first module which should be tensorflow + symbol_id_split = symbol_id.split('.')[1:] + symbol = tf + for sym in symbol_id_split: + symbol = getattr(symbol, sym) + return symbol + + +def _IsGenModule(module_name): + if not module_name: + return False + module_name_split = module_name.split('.') + return module_name_split[-1].startswith('gen_') + + class ApiCompatibilityTest(test.TestCase): def __init__(self, *args, **kwargs): @@ -229,6 +262,150 @@ class ApiCompatibilityTest(test.TestCase): update_goldens=FLAGS.update_goldens) +class ApiDefTest(test.TestCase): + + def __init__(self, *args, **kwargs): + super(ApiDefTest, self).__init__(*args, **kwargs) + self._first_cap_pattern = re.compile('(.)([A-Z][a-z]+)') + self._all_cap_pattern = re.compile('([a-z0-9])([A-Z])') + + def _GenerateLowerCaseOpName(self, op_name): + lower_case_name = self._first_cap_pattern.sub(r'\1_\2', op_name) + return self._all_cap_pattern.sub(r'\1_\2', lower_case_name).lower() + + def _CreatePythonApiDef(self, base_api_def, endpoint_names): + """Creates Python ApiDef that overrides base_api_def if needed. + + Args: + base_api_def: (api_def_pb2.ApiDef) base ApiDef instance. + endpoint_names: List of Python endpoint names. + + Returns: + api_def_pb2.ApiDef instance with overrides for base_api_def + if module.name endpoint is different from any existing + endpoints in base_api_def. Otherwise, returns None. + """ + endpoint_names_set = set(endpoint_names) + base_endpoint_names_set = { + self._GenerateLowerCaseOpName(endpoint.name) + for endpoint in base_api_def.endpoint} + + if endpoint_names_set == base_endpoint_names_set: + return None # All endpoints are the same + + api_def = api_def_pb2.ApiDef() + api_def.graph_op_name = base_api_def.graph_op_name + + for endpoint_name in sorted(endpoint_names): + new_endpoint = api_def.endpoint.add() + new_endpoint.name = endpoint_name + + return api_def + + def _GetBaseApiMap(self): + """Get a map from graph op name to its base ApiDef. + + Returns: + Dictionary mapping graph op name to corresponding ApiDef. + """ + # Convert base ApiDef in Multiline format to Proto format. + converted_base_api_dir = os.path.join( + test.get_temp_dir(), 'temp_base_api_defs') + subprocess.check_call( + [os.path.join(resource_loader.get_root_dir_with_all_resources(), + _CONVERT_FROM_MULTILINE_SCRIPT), + _BASE_API_DIR, converted_base_api_dir]) + + name_to_base_api_def = {} + base_api_files = file_io.get_matching_files( + os.path.join(converted_base_api_dir, 'api_def_*.pbtxt')) + for base_api_file in base_api_files: + if file_io.file_exists(base_api_file): + api_defs = api_def_pb2.ApiDefs() + text_format.Merge( + file_io.read_file_to_string(base_api_file), api_defs) + for api_def in api_defs.op: + lower_case_name = self._GenerateLowerCaseOpName(api_def.graph_op_name) + name_to_base_api_def[lower_case_name] = api_def + return name_to_base_api_def + + @unittest.skipUnless( + sys.version_info.major == 2 and os.uname()[0] == 'Linux', + 'API compabitility test goldens are generated using python2 on Linux.') + def testAPIDefCompatibility(self): + # Get base ApiDef + name_to_base_api_def = self._GetBaseApiMap() + # Extract Python API + visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor() + public_api_visitor = public_api.PublicAPIVisitor(visitor) + public_api_visitor.do_not_descend_map['tf'].append('contrib') + traverse.traverse(tf, public_api_visitor) + proto_dict = visitor.GetProtos() + + # Map from first character of op name to Python ApiDefs. + api_def_map = defaultdict(api_def_pb2.ApiDefs) + # We need to override all endpoints even if 1 endpoint differs from base + # ApiDef. So, we first create a map from an op to all its endpoints. + op_to_endpoint_name = defaultdict(list) + + # Generate map from generated python op to endpoint names. + for public_module, value in proto_dict.items(): + module_obj = _GetSymbol(public_module) + for sym in value.tf_module.member_method: + obj = getattr(module_obj, sym.name) + + # Check if object is defined in gen_* module. That is, + # the object has been generated from OpDef. + if hasattr(obj, '__module__') and _IsGenModule(obj.__module__): + if obj.__name__ not in name_to_base_api_def: + # Symbol might be defined only in Python and not generated from + # C++ api. + continue + relative_public_module = public_module[len('tensorflow.'):] + full_name = (relative_public_module + '.' + sym.name + if relative_public_module else sym.name) + op_to_endpoint_name[obj].append(full_name) + + # Generate Python ApiDef overrides. + for op, endpoint_names in op_to_endpoint_name.items(): + api_def = self._CreatePythonApiDef( + name_to_base_api_def[op.__name__], endpoint_names) + if api_def: + api_defs = api_def_map[op.__name__[0].upper()] + api_defs.op.extend([api_def]) + + for key in _ALPHABET: + # Get new ApiDef for the given key. + new_api_defs_str = '' + if key in api_def_map: + new_api_defs = api_def_map[key] + new_api_defs.op.sort(key=attrgetter('graph_op_name')) + new_api_defs_str = str(new_api_defs) + + # Get current ApiDef for the given key. + api_defs_file_path = os.path.join( + _PYTHON_API_DIR, 'api_def_%s.pbtxt' % key) + old_api_defs_str = '' + if file_io.file_exists(api_defs_file_path): + old_api_defs_str = file_io.read_file_to_string(api_defs_file_path) + + if old_api_defs_str == new_api_defs_str: + continue + + if FLAGS.update_goldens: + if not new_api_defs_str: + logging.info('Deleting %s...' % api_defs_file_path) + file_io.delete_file(api_defs_file_path) + else: + logging.info('Updating %s...' % api_defs_file_path) + file_io.write_string_to_file(api_defs_file_path, new_api_defs_str) + else: + self.assertMultiLineEqual( + old_api_defs_str, new_api_defs_str, + 'To update golden API files, run api_compatibility_test locally ' + 'with --update_goldens=True flag.') + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( diff --git a/tensorflow/tools/api/tests/convert_from_multiline.cc b/tensorflow/tools/api/tests/convert_from_multiline.cc new file mode 100644 index 0000000000..5c5aaa4f06 --- /dev/null +++ b/tensorflow/tools/api/tests/convert_from_multiline.cc @@ -0,0 +1,63 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Converts all *.pbtxt files in a directory from Multiline to proto format. +#include "tensorflow/core/framework/op_gen_lib.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/init_main.h" + +namespace tensorflow { + +namespace { +constexpr char kApiDefFilePattern[] = "*.pbtxt"; + +Status ConvertFilesFromMultiline(const string& input_dir, + const string& output_dir) { + Env* env = Env::Default(); + + const string file_pattern = io::JoinPath(input_dir, kApiDefFilePattern); + std::vector matching_paths; + TF_CHECK_OK(env->GetMatchingPaths(file_pattern, &matching_paths)); + + if (!env->IsDirectory(output_dir).ok()) { + TF_RETURN_IF_ERROR(env->CreateDir(output_dir)); + } + + for (const auto& path : matching_paths) { + string contents; + TF_RETURN_IF_ERROR(tensorflow::ReadFileToString(env, path, &contents)); + contents = tensorflow::PBTxtFromMultiline(contents); + string output_path = io::JoinPath(output_dir, io::Basename(path)); + // Write contents to output_path + TF_RETURN_IF_ERROR( + tensorflow::WriteStringToFile(env, output_path, contents)); + } + return Status::OK(); +} +} // namespace +} // namespace tensorflow + +int main(int argc, char* argv[]) { + tensorflow::port::InitMain(argv[0], &argc, &argv); + + const std::string usage = + "Usage: convert_from_multiline input_dir output_dir"; + if (argc != 3) { + std::cerr << usage << std::endl; + return -1; + } + TF_CHECK_OK(tensorflow::ConvertFilesFromMultiline(argv[1], argv[2])); + return 0; +} -- GitLab From ba49d85832918837c2d568545f73cc3b2e47763c Mon Sep 17 00:00:00 2001 From: Bjarke Hammersholt Roune Date: Fri, 20 Oct 2017 19:44:11 -0700 Subject: [PATCH 240/573] Slight change to reduce_test to avoid generating inf, which was triggering an inf detector unnecessarily. PiperOrigin-RevId: 172965466 --- tensorflow/compiler/xla/tests/reduce_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc index b48b3a2bdb..794e5a4920 100644 --- a/tensorflow/compiler/xla/tests/reduce_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_test.cc @@ -457,7 +457,7 @@ XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) { const Shape input_shape = ShapeUtil::MakeShape(F32, {rows, 2, cols / 2}); auto input = builder.Parameter(0, input_shape, "input"); auto zero = builder.ConstantR0(0.0); - auto log_ = builder.Log(input); + auto log_ = builder.Tanh(input); auto reshape = builder.Reshape(log_, {rows, cols}); builder.Reduce(reshape, zero, add_f32, /*dimensions_to_reduce=*/{0}); @@ -473,7 +473,7 @@ XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) { for (int64 colno = 0; colno < cols / 2; ++colno) { float column_sum = 0; for (int64 rowno = 0; rowno < rows; ++rowno) { - column_sum += log(input_data(rowno, major, colno)); + column_sum += tanh(input_data(rowno, major, colno)); } expected.push_back(column_sum); } -- GitLab From 9d55c249c18745fdd2e4e50b8faa3eef2aac4f90 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 21 Oct 2017 10:06:45 -0700 Subject: [PATCH 241/573] Fix doc in TF_CALL_ when invoked in mobile platform (#13881) * Fix doc in TF_CALL_ when defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__) This is a small doc fix that includes bool as part of the types that is supported in mobile (IS_MOBILE_PLATFORM && !__ANDROID_TYPES_FULL__), as bool is clearly invoked in the following define. Signed-off-by: Yong Tang * Also add bool to android full version. Signed-off-by: Yong Tang --- tensorflow/core/framework/register_types.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h index 61e722e57b..c31ab18cc1 100644 --- a/tensorflow/core/framework/register_types.h +++ b/tensorflow/core/framework/register_types.h @@ -87,7 +87,7 @@ limitations under the License. #elif defined(__ANDROID_TYPES_FULL__) -// Only half, float, int32, int64, and quantized types are supported. +// Only half, float, int32, int64, bool, and quantized types are supported. #define TF_CALL_float(m) m(float) #define TF_CALL_double(m) #define TF_CALL_int32(m) m(::tensorflow::int32) @@ -117,7 +117,7 @@ limitations under the License. #else // defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__) -// Only float and int32 are supported. +// Only float, int32, and bool are supported. #define TF_CALL_float(m) m(float) #define TF_CALL_double(m) #define TF_CALL_int32(m) m(::tensorflow::int32) -- GitLab From a699458107fd2c6960cda61c6bfef4cd03025887 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 21 Oct 2017 11:49:50 -0700 Subject: [PATCH 242/573] Update pin for bazel-toolchains to latest version PiperOrigin-RevId: 173002530 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index a863aa18dd..8ba8748aae 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -742,9 +742,9 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "bazel_toolchains", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", - # "https://github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz", + # "https://github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz", ], - sha256 = "46187270ca04ff8109980f45c3438fabfe48695e163789096eb82ee097ffe685", - strip_prefix = "bazel-toolchains-b2b4b38433bf2d1159360855ea4004378308711b", + sha256 = "d58bb2d6c8603f600d522b6104d6192a65339aa26cbba9f11ff5c4b36dedb928", + strip_prefix = "bazel-toolchains-af4681c3d19f063f090222ec3d04108c4e0ca255", ) -- GitLab From d1183ca6a245cd0b498c46fd1079909ebc4abc3a Mon Sep 17 00:00:00 2001 From: Vijay Vasudevan Date: Sat, 21 Oct 2017 13:43:01 -0700 Subject: [PATCH 243/573] Give each variable a unique name in accumulate_n_v2_eager_test. (#13886) --- .../contrib/framework/python/ops/accumulate_n_v2_eager_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py index 5f086ea8cc..c2229bb8ad 100644 --- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py +++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py @@ -65,7 +65,7 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase): num_inputs = 3 input_vars = [ resource_variable_ops.ResourceVariable(10.0 * np.random.random(), - name="t1") + name="t%d" % i) for i in range(0, num_inputs) ] -- GitLab From b927df57f0c09ea62a855795e340d6daf70553df Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 21 Oct 2017 22:05:18 -0700 Subject: [PATCH 244/573] Update protobuf.cmake to b04e5cba356212e4e8c66c61bbe0c3a20537c5b9 (#13893) This fix tries to address the issue raised in 8187 where protobuf.cmake used different version as bazel. The reason for discrepancy was due to the fact that a customerized protobuf was needed with Windows patch. Since the patch has been merged in (https://github.com/google/protobuf/pull/2203), it makes sense to update protobuf.cmake so that the same version of cmake is used. This fix fixes 8187. Signed-off-by: Yong Tang --- tensorflow/contrib/cmake/external/protobuf.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake index d600d8c3c0..1e300e21df 100644 --- a/tensorflow/contrib/cmake/external/protobuf.cmake +++ b/tensorflow/contrib/cmake/external/protobuf.cmake @@ -15,8 +15,8 @@ include (ExternalProject) set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src) -set(PROTOBUF_URL https://github.com/mrry/protobuf.git) # Includes MSVC fix. -set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05) +set(PROTOBUF_URL https://github.com/google/protobuf.git) +set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9) if(WIN32) set(protobuf_STATIC_LIBRARIES -- GitLab From 17096081eed7881c0b8ce3c32b5e9795619e27bb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 21 Oct 2017 22:14:40 -0700 Subject: [PATCH 245/573] Improve resize_bicubic performance by reorganizing loops (#13840) * Improve resize_bicubic performance by reorganizing loops This fix tries to address the issue raised in 13693 where performance of `resize_bicubic` is not on par with opencv. This fix rearranges the loops so that it is the same for num_channel=40 and num_channel=3: Pre-fix: ``` CHANNEL=40 opencv: 145.08ms tf: 314.26ms CHANNEL=3 opencv: 11.95ms tf: 8.95ms ``` Post-fix: ``` CHANNEL=40 opencv: 144.25ms tf: 214.55ms CHANNEL=3 opencv: 11.78ms tf: 14.07ms ``` This fix fixes 13693. Signed-off-by: Yong Tang * Keep special handling of `num_channels=3` for `resize_bicubic` This commit keeps special handling of `num_channels=3` for `resize_bicubic`: Without special handling: ``` opencv: 11.78ms tf: 14.07ms ``` With special handling: ``` opencv: 11.74ms tf: 9.46ms ``` Signed-off-by: Yong Tang * Expand Benchmark test for resize_bicubic Signed-off-by: Yong Tang * Update from review feedback. Signed-off-by: Yong Tang --- tensorflow/core/kernels/resize_bicubic_op.cc | 85 +++++++++++-------- .../core/kernels/resize_bicubic_op_test.cc | 20 ++++- 2 files changed, 67 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/kernels/resize_bicubic_op.cc b/tensorflow/core/kernels/resize_bicubic_op.cc index 1c43e77e7c..1a9cf4c640 100644 --- a/tensorflow/core/kernels/resize_bicubic_op.cc +++ b/tensorflow/core/kernels/resize_bicubic_op.cc @@ -20,7 +20,6 @@ limitations under the License. #include #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" @@ -29,6 +28,7 @@ limitations under the License. #include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" namespace tensorflow { namespace { @@ -235,6 +235,7 @@ inline void interpolate_with_caching( const T* input_b_ptr = input_data.data(); float* output_y_ptr = output_data.data(); + std::vector cached_value(num_channels == 3 ? 0 : 4 * num_channels, 0); for (int64 b = 0; b < resizer_state.batch_size; ++b, input_b_ptr += in_batch_width) { @@ -248,6 +249,7 @@ inline void interpolate_with_caching( const T* y_ptr_1 = input_b_ptr + y_wai.index_1 * in_row_width; const T* y_ptr_2 = input_b_ptr + y_wai.index_2 * in_row_width; const T* y_ptr_3 = input_b_ptr + y_wai.index_3 * in_row_width; + if (num_channels == 3) { // Manually unroll case of 3 channels. float cached_value_0[4] = {0}; @@ -330,48 +332,61 @@ inline void interpolate_with_caching( x_wai.weight_2, x_wai.weight_3); } } else { - for (int64 c = 0; c < num_channels; ++c) { - float cached_value[4] = {0}; - for (int64 x = 0; x < resizer_state.out_width; ++x) { - const WeightsAndIndices& x_wai = x_wais[x]; - // Shift values in cached_value to fill first 'advance' values. - switch (x_wai.advance) { - case 3: - cached_value[0] = cached_value[1]; - cached_value[1] = cached_value[2]; - cached_value[2] = cached_value[3]; - break; - case 2: - cached_value[0] = cached_value[2]; - cached_value[1] = cached_value[3]; - break; - case 1: { - cached_value[0] = cached_value[3]; - break; + for (int64 x = 0; x < resizer_state.out_width; ++x) { + const WeightsAndIndices& x_wai = x_wais[x]; + // Shift values in cached_value to fill first 'advance' values. + switch (x_wai.advance) { + case 3: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = cached_value[4 * c + 1]; + cached_value[4 * c + 1] = cached_value[4 * c + 2]; + cached_value[4 * c + 2] = cached_value[4 * c + 3]; + } + break; + case 2: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = cached_value[4 * c + 2]; + cached_value[4 * c + 1] = cached_value[4 * c + 3]; + } + break; + case 1: { + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = cached_value[4 * c + 3]; } + break; } + } - // Set the remaining '4-advance' values by computing. - switch (x_wai.advance) { - case 0: - cached_value[0] = ComputeYInterpolation( + // Set the remaining '4-advance' values by computing. + switch (x_wai.advance) { + case 0: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = ComputeYInterpolation( 0, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - TF_FALLTHROUGH_INTENDED; - case 1: - cached_value[1] = ComputeYInterpolation( + } + TF_FALLTHROUGH_INTENDED; + case 1: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 1] = ComputeYInterpolation( 1, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - TF_FALLTHROUGH_INTENDED; - case 2: - cached_value[2] = ComputeYInterpolation( + } + TF_FALLTHROUGH_INTENDED; + case 2: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 2] = ComputeYInterpolation( 2, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - TF_FALLTHROUGH_INTENDED; - case 3: - cached_value[3] = ComputeYInterpolation( + } + TF_FALLTHROUGH_INTENDED; + case 3: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 3] = ComputeYInterpolation( 3, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - break; - } + } + break; + } + for (int64 c = 0; c < num_channels; ++c) { output_y_ptr[x * num_channels + c] = - Compute(cached_value, x_wai.weight_0, x_wai.weight_1, + Compute(&cached_value[4 * c], x_wai.weight_0, x_wai.weight_1, x_wai.weight_2, x_wai.weight_3); } } diff --git a/tensorflow/core/kernels/resize_bicubic_op_test.cc b/tensorflow/core/kernels/resize_bicubic_op_test.cc index ae14d2804e..9e10fec423 100644 --- a/tensorflow/core/kernels/resize_bicubic_op_test.cc +++ b/tensorflow/core/kernels/resize_bicubic_op_test.cc @@ -251,14 +251,15 @@ TEST_F(ResizeBicubicOpTest, TestAreaRandomDataSeveralInputsSizes4Channels) { RunManyRandomTests(4); } -static Graph* ResizeBicubic(int batch_size, int size, int channels) { +static Graph* ResizeBicubic(int batch_size, int size, int channels, + float scale_y = 0.3, float scale_x = 0.7) { Graph* g = new Graph(OpRegistry::Global()); Tensor input(DT_FLOAT, TensorShape({batch_size, size, size, channels})); input.flat().setRandom(); Tensor shape(DT_INT32, TensorShape({2})); auto shape_t = shape.flat(); - shape_t(0) = 0.3 * size; - shape_t(1) = 0.7 * size; + shape_t(0) = scale_y * size; + shape_t(1) = scale_x * size; test::graph::Binary(g, "ResizeBicubic", test::graph::Constant(g, input), test::graph::Constant(g, shape)); return g; @@ -285,4 +286,17 @@ BM_ResizeBicubicDev(32, 128, 3); BM_ResizeBicubicDev(32, 512, 3); BM_ResizeBicubicDev(32, 1024, 3); +#define BM_ResizeBicubicExpand(BATCH, SIZE, CHANNELS) \ + static void BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS(int iters) { \ + testing::ItemsProcessed(static_cast(iters) * BATCH * SIZE * SIZE * \ + CHANNELS * 8 * 8); \ + test::Benchmark("cpu", ResizeBicubic(BATCH, SIZE, CHANNELS, 8, 8)) \ + .Run(iters); \ + } \ + BENCHMARK(BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS); + +BM_ResizeBicubicExpand(12, 48, 1); +BM_ResizeBicubicExpand(12, 48, 3); +BM_ResizeBicubicExpand(12, 48, 40); + } // end namespace tensorflow -- GitLab From 1c1dad105a57bb13711492a8ba5ab9d10c91b5df Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 21 Oct 2017 23:01:42 -0700 Subject: [PATCH 246/573] Add int64 axis support for reduction ops. (#13891) * Add int64 axis support for reduction ops. This fix is a follow up to PR 13863. In PR 13863 the program crash is fixed if int64 axis is passed to reduction ops, e.g. reduce_sum, reduce_max, etc. However, 13863 does not process the case of int64 support, it merely fixes the crash. This fix adds the support for int64 axis of reduction ops. Signed-off-by: Yong Tang * Add int64 axis support for mean, prod, sum Signed-off-by: Yong Tang * Add int64 axis support for min and max. Signed-off-by: Yong Tang * Add int64 axis support for reduce_all and reduce_any Signed-off-by: Yong Tang * Add test cases for int64 axis support of reduce_any and reduce_all Signed-off-by: Yong Tang --- tensorflow/core/kernels/reduction_ops_all.cc | 16 +++- tensorflow/core/kernels/reduction_ops_any.cc | 16 +++- .../core/kernels/reduction_ops_common.cc | 22 +++-- .../core/kernels/reduction_ops_common.h | 27 +++--- tensorflow/core/kernels/reduction_ops_max.cc | 90 +++++++++++++------ tensorflow/core/kernels/reduction_ops_mean.cc | 68 +++++++++----- tensorflow/core/kernels/reduction_ops_min.cc | 90 +++++++++++++------ tensorflow/core/kernels/reduction_ops_prod.cc | 68 +++++++++----- tensorflow/core/kernels/reduction_ops_sum.cc | 90 +++++++++++++------ .../python/kernel_tests/reduction_ops_test.py | 52 +++++++++++ 10 files changed, 391 insertions(+), 148 deletions(-) diff --git a/tensorflow/core/kernels/reduction_ops_all.cc b/tensorflow/core/kernels/reduction_ops_all.cc index 41abc2b957..4a34c4ef51 100644 --- a/tensorflow/core/kernels/reduction_ops_all.cc +++ b/tensorflow/core/kernels/reduction_ops_all.cc @@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_CPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("All") + .TypeConstraint("Tidx") + .Device(DEVICE_CPU) + .HostMemory("reduction_indices"), + ReductionOp); #if GOOGLE_CUDA REGISTER_KERNEL_BUILDER( @@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_GPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("All") + .TypeConstraint("Tidx") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices"), + ReductionOp); #endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_any.cc b/tensorflow/core/kernels/reduction_ops_any.cc index a2087cc3b7..6c0519de95 100644 --- a/tensorflow/core/kernels/reduction_ops_any.cc +++ b/tensorflow/core/kernels/reduction_ops_any.cc @@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_CPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("Any") + .TypeConstraint("Tidx") + .Device(DEVICE_CPU) + .HostMemory("reduction_indices"), + ReductionOp); #if GOOGLE_CUDA REGISTER_KERNEL_BUILDER( @@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_GPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("Any") + .TypeConstraint("Tidx") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices"), + ReductionOp); #endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_common.cc b/tensorflow/core/kernels/reduction_ops_common.cc index 5eba4288ac..8daab0d6be 100644 --- a/tensorflow/core/kernels/reduction_ops_common.cc +++ b/tensorflow/core/kernels/reduction_ops_common.cc @@ -57,13 +57,12 @@ gtl::InlinedVector ReductionHelper::permutation() { return perm; } -Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis, - const bool keep_dims) { - // bitmap[i] indicates whether to reduce data along i-th axis. - gtl::InlinedVector bitmap(data.dims(), false); - auto axis_vec = axis.flat(); +template +Status SimplifyHelper(const Tensor& data, const Tensor& axis, + gtl::InlinedVector& bitmap) { + auto axis_vec = axis.flat(); for (int64 i = 0; i < axis.NumElements(); ++i) { - int32 index = axis_vec(i); + Tperm index = axis_vec(i); if (index < -data.dims() || index >= data.dims()) { return errors::InvalidArgument("Invalid reduction dimension (", index, " for input with ", data.dims(), @@ -72,7 +71,18 @@ Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis, index = (index + data.dims()) % data.dims(); bitmap[index] = true; } + return Status::OK(); +} +Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis, + const bool keep_dims) { + // bitmap[i] indicates whether to reduce data along i-th axis. + gtl::InlinedVector bitmap(data.dims(), false); + if (axis.dtype() == DT_INT32) { + TF_RETURN_IF_ERROR(SimplifyHelper(data, axis, bitmap)); + } else { + TF_RETURN_IF_ERROR(SimplifyHelper(data, axis, bitmap)); + } // Output tensor's dim sizes. out_shape_.clear(); for (int i = 0; i < data.dims(); ++i) { diff --git a/tensorflow/core/kernels/reduction_ops_common.h b/tensorflow/core/kernels/reduction_ops_common.h index 71af9d88dc..9da992ccd1 100644 --- a/tensorflow/core/kernels/reduction_ops_common.h +++ b/tensorflow/core/kernels/reduction_ops_common.h @@ -25,6 +25,7 @@ limitations under the License. #include "third_party/eigen3/Eigen/Core" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -42,7 +43,7 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL template struct Constants { @@ -68,11 +69,13 @@ struct ConstantsBase { const Eigen::IndexList> kOne; const Eigen::IndexList, Eigen::type2index<2>> kZeroTwo; }; -template<> struct Constants : ConstantsBase{}; +template <> +struct Constants : ConstantsBase {}; #ifdef TENSORFLOW_USE_SYCL -template<> struct Constants : ConstantsBase{}; -#endif // TENSORFLOW_USE_SYCL -#endif // EIGEN_HAS_INDEX_LIST +template <> +struct Constants : ConstantsBase {}; +#endif // TENSORFLOW_USE_SYCL +#endif // EIGEN_HAS_INDEX_LIST class ReductionHelper { public: @@ -131,12 +134,13 @@ class ReductionHelper { // For operations where the output is a reduction function along some // dimensions of the input. -template +template class ReductionOp : public OpKernel { public: explicit ReductionOp(OpKernelConstruction* ctx) : OpKernel(ctx) { const DataType dt = DataTypeToEnum::v(); - OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, DT_INT32}, {dt})); + const DataType pt = DataTypeToEnum::v(); + OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, pt}, {dt})); OP_REQUIRES_OK(ctx, ctx->GetAttr("keep_dims", &keep_dims_)); } @@ -266,20 +270,19 @@ struct ReduceFunctorBase { } template - static void FillIdentity(const Device& d, OUT_T out, - const Reducer& reducer) { + static void FillIdentity(const Device& d, OUT_T out, const Reducer& reducer) { FillIdentityEigenImpl(d, out, reducer); } }; template struct ReduceFunctor - : ReduceFunctorBase{}; + : ReduceFunctorBase {}; #if TENSORFLOW_USE_SYCL template struct ReduceFunctor - : ReduceFunctorBase{}; -#endif // TENSORFLOW_USE_SYCL + : ReduceFunctorBase {}; +#endif // TENSORFLOW_USE_SYCL } // namespace functor } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_max.cc b/tensorflow/core/kernels/reduction_ops_max.cc index 4ca5c11a48..9cf953f4bf 100644 --- a/tensorflow/core/kernels/reduction_ops_max.cc +++ b/tensorflow/core/kernels/reduction_ops_max.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Max") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Max") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_GPU_KERNELS(float); REGISTER_GPU_KERNELS(double); REGISTER_GPU_KERNELS(int64); @@ -52,21 +65,37 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Max") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_GPU_KERNELS #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Max") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Max") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Max") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); @@ -78,8 +107,17 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Max") + .Device(DEVICE_SYCL) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_mean.cc b/tensorflow/core/kernels/reduction_ops_mean.cc index 5b01de8ddb..f61589f913 100644 --- a/tensorflow/core/kernels/reduction_ops_mean.cc +++ b/tensorflow/core/kernels/reduction_ops_mean.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Mean") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Mean") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); @@ -45,17 +58,24 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Mean") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc index 1e394bea41..807ac0a456 100644 --- a/tensorflow/core/kernels/reduction_ops_min.cc +++ b/tensorflow/core/kernels/reduction_ops_min.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Min") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Min") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_GPU_KERNELS(float); REGISTER_GPU_KERNELS(double); @@ -51,21 +64,37 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Min") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_GPU_KERNELS #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Min") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Min") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Min") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); @@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Min") + .Device(DEVICE_SYCL) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_prod.cc b/tensorflow/core/kernels/reduction_ops_prod.cc index 33f6ae6bae..e9b23df746 100644 --- a/tensorflow/core/kernels/reduction_ops_prod.cc +++ b/tensorflow/core/kernels/reduction_ops_prod.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Prod") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Prod") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_int32(REGISTER_GPU_KERNELS); TF_CALL_complex64(REGISTER_GPU_KERNELS); @@ -46,18 +59,25 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Prod") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(int32); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_sum.cc b/tensorflow/core/kernels/reduction_ops_sum.cc index c1f4f3475a..5318d8c133 100644 --- a/tensorflow/core/kernels/reduction_ops_sum.cc +++ b/tensorflow/core/kernels/reduction_ops_sum.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Sum") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Sum") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); @@ -53,19 +66,35 @@ REGISTER_KERNEL_BUILDER( .HostMemory("input") .HostMemory("output") .HostMemory("reduction_indices"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Sum") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .TypeConstraint("Tidx") + .HostMemory("input") + .HostMemory("output") + .HostMemory("reduction_indices"), + ReductionOp>); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Sum") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Sum") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Sum") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); @@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER( .HostMemory("input") .HostMemory("output") .HostMemory("reduction_indices"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Sum") + .Device(DEVICE_SYCL) + .TypeConstraint("T") + .TypeConstraint("Tidx") + .HostMemory("input") + .HostMemory("output") + .HostMemory("reduction_indices"), + ReductionOp>); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index c794351fe9..2dc65b1384 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -163,6 +163,13 @@ class SumReductionTest(BaseReductionTest): reduction_axes = tuple(reduction_axes) return np.sum(x, axis=reduction_axes, keepdims=keep_dims) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -193,6 +200,7 @@ class SumReductionTest(BaseReductionTest): tf_out_mean = sess.run(tf_mean) self.assertAllClose(tf_out_mean, 1.) + def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) @@ -369,6 +377,13 @@ class MeanReductionTest(BaseReductionTest): return np_sum // count return np_sum / count + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -435,6 +450,13 @@ class ProdReductionTest(BaseReductionTest): reduction_axes = tuple(reduction_axes) return np.prod(x, axis=reduction_axes, keepdims=keep_dims) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -531,6 +553,13 @@ class MinReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -637,6 +666,13 @@ class MaxReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -757,6 +793,14 @@ class AllReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_all([True, True], + constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, True) + def testAll3D(self): # Create a 3D array of bools and reduce across all possible # dimensions @@ -798,6 +842,14 @@ class AnyReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_any([True, True], + constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, True) + def testAll3D(self): # Create a 3D array of bools and reduce across all possible # dimensions -- GitLab From c9cb5a58d5e174e9870c40328d6be427ccf8be54 Mon Sep 17 00:00:00 2001 From: formath Date: Sun, 22 Oct 2017 14:02:50 +0800 Subject: [PATCH 247/573] protobuf lib path bug fix for benckmark on osx (#13878) --- tensorflow/contrib/makefile/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 3dcff3d4a3..e970e50d2e 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -190,6 +190,10 @@ LIBFLAGS := # If we're on OS X, make sure that globals aren't stripped out. ifeq ($(TARGET),OSX) +ifeq ($(HAS_GEN_HOST_PROTOC),true) + LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib + export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib +endif LDFLAGS += -all_load endif # Make sure that we don't strip global constructors on Linux. -- GitLab From bfa4ec194a595f7ed466f2e5af391c81e98786bc Mon Sep 17 00:00:00 2001 From: Tayo Oguntebi <10927929+tayo@users.noreply.github.com> Date: Sat, 21 Oct 2017 23:08:17 -0700 Subject: [PATCH 248/573] Update node_def.proto comments (#13874) The device field had outdated comments. Note: We could consider adding tpu as an example here, e.g. "gpu" | "cpu" | "tpu". Thoughts? --- tensorflow/core/framework/node_def.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto index 53aa03108a..1fd2e50b51 100644 --- a/tensorflow/core/framework/node_def.proto +++ b/tensorflow/core/framework/node_def.proto @@ -35,7 +35,7 @@ message NodeDef { // CONSTRAINT ::= ("job:" JOB_NAME) // | ("replica:" [1-9][0-9]*) // | ("task:" [1-9][0-9]*) - // | ( ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") ) + // | ("device:" ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") ) // // Valid values for this string include: // * "/job:worker/replica:0/task:1/device:GPU:3" (full specification) -- GitLab From 40c475b48c091a70ad8061c1508dff6ded2d2af6 Mon Sep 17 00:00:00 2001 From: formath Date: Mon, 23 Oct 2017 03:05:08 +0800 Subject: [PATCH 249/573] add segment_reduction_ops to tf_op_files (#13901) --- tensorflow/contrib/makefile/tf_op_files.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt index a8690a04ad..8b77c99cb5 100644 --- a/tensorflow/contrib/makefile/tf_op_files.txt +++ b/tensorflow/contrib/makefile/tf_op_files.txt @@ -264,3 +264,4 @@ tensorflow/core/kernels/spacetobatch_functor.cc tensorflow/core/kernels/spacetobatch_op.cc tensorflow/core/kernels/batchtospace_op.cc tensorflow/core/kernels/warn_about_ints.cc +tensorflow/core/kernels/segment_reduction_ops.cc -- GitLab From fd8d517b97da8b41ad4088b2fc68080393f26b55 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Sun, 22 Oct 2017 16:48:19 -0700 Subject: [PATCH 250/573] Add tests for convolution 1D RELNOTES: n/a PiperOrigin-RevId: 173060283 --- .../compiler/xla/tests/convolution_test.cc | 79 ++++++++++++++----- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc index a7089c2897..0cc2e5fb7e 100644 --- a/tensorflow/compiler/xla/tests/convolution_test.cc +++ b/tensorflow/compiler/xla/tests/convolution_test.cc @@ -508,21 +508,35 @@ XLA_TEST_F(ConvolutionTest, Convolve2D_1x3x3x5_3x3x5x5_Valid) { error_spec_); } -XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) { +struct Convolve1DTestParam { + int64 input_feature; + int64 output_feature; + int64 batch; + int64 window_size; + int64 num_windows; +}; + +class Convolve1D1WindowTest + : public ConvolutionTest, + public ::testing::WithParamInterface {}; + +XLA_TEST_P(Convolve1D1WindowTest, Convolve1D1Window) { ComputationBuilder builder(client_, TestName()); - int64 output_feature = 1; - int64 input_feature = 64; - int64 batch = 1; - int64 length = 1; - std::vector input_dims = {batch, 4 + length - 1, input_feature}; - std::vector filter_dims = {4, input_feature, output_feature}; + int64 input_feature = GetParam().input_feature; + int64 output_feature = GetParam().output_feature; + int64 batch = GetParam().batch; + int64 num_windows = GetParam().num_windows; + int64 window_size = GetParam().window_size; + std::vector input_dims = {batch, window_size + num_windows - 1, + input_feature}; + std::vector filter_dims = {window_size, input_feature, output_feature}; Shape input_shape = ShapeUtil::MakeShape(F32, input_dims); Shape filter_shape = ShapeUtil::MakeShape(F32, filter_dims); { auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - // Tensorflow dimension numbers for 2D convolution. + // Tensorflow dimension numbers for 1D convolution. ConvolutionDimensionNumbers dnums; dnums.set_input_batch_dimension(0); dnums.set_output_batch_dimension(0); @@ -538,28 +552,57 @@ XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) { } std::vector input_elems(ShapeUtil::ElementsIn(input_shape), 1.0); - // std::iota(input_elems.begin(), input_elems.end(), 1.0f); auto input_r1 = Literal::CreateR1(input_elems); - auto input_r4 = input_r1->Reshape(input_dims).ConsumeValueOrDie(); + auto input_r3 = input_r1->Reshape(input_dims).ConsumeValueOrDie(); std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), 1.0); - // std::iota(filter_elems.begin(), filter_elems.end(), 1.0f); auto filter_r1 = Literal::CreateR1(filter_elems); - auto filter_r4 = filter_r1->Reshape(filter_dims).ConsumeValueOrDie(); + auto filter_r3 = filter_r1->Reshape(filter_dims).ConsumeValueOrDie(); - std::vector expect_elems(batch * output_feature * length, 256); + std::vector expect_elems(batch * output_feature * num_windows, + window_size * input_feature); auto expected_r1 = Literal::CreateR1(expect_elems); - auto expected_r4 = - expected_r1->Reshape({batch, length, output_feature}).ConsumeValueOrDie(); + auto expected_r3 = expected_r1->Reshape({batch, num_windows, output_feature}) + .ConsumeValueOrDie(); - auto input_literal = client_->TransferToServer(*input_r4).ConsumeValueOrDie(); + auto input_literal = client_->TransferToServer(*input_r3).ConsumeValueOrDie(); auto filter_literal = - client_->TransferToServer(*filter_r4).ConsumeValueOrDie(); - ComputeAndCompareLiteral(&builder, *expected_r4, + client_->TransferToServer(*filter_r3).ConsumeValueOrDie(); + ComputeAndCompareLiteral(&builder, *expected_r3, {input_literal.get(), filter_literal.get()}, error_spec_); } +INSTANTIATE_TEST_CASE_P( + Convolve1D1WindowTest_Instantiation, Convolve1D1WindowTest, + ::testing::Values(Convolve1DTestParam{1, 1, 1, 1, 2}, + Convolve1DTestParam{160, 1, 1, 5, 1}, + Convolve1DTestParam{24, 1, 1, 20, 1}, + Convolve1DTestParam{30, 1, 1, 20, 1}, + Convolve1DTestParam{23, 1, 1, 20, 20}, + Convolve1DTestParam{25, 1, 1, 20, 1}, + Convolve1DTestParam{24, 1, 1, 10, 5}, + Convolve1DTestParam{160, 1, 1, 10, 1}, + Convolve1DTestParam{255, 1, 1, 3, 1}, + Convolve1DTestParam{130, 1, 1, 1, 3}, + Convolve1DTestParam{64, 1, 1, 1, 1}, + Convolve1DTestParam{128, 1, 1, 1, 1}, + Convolve1DTestParam{139, 1, 1, 128, 1}, + Convolve1DTestParam{1, 10, 10, 1, 10}, + Convolve1DTestParam{1, 10, 130, 1, 2}, + Convolve1DTestParam{1, 10, 130, 1, 1}, + Convolve1DTestParam{1, 64, 64, 1, 10}, + Convolve1DTestParam{1, 65, 65, 1, 1}, + Convolve1DTestParam{1, 128, 128, 1, 1}, + Convolve1DTestParam{128, 128, 128, 128, 1}, + Convolve1DTestParam{1, 128, 128, 1, 1}, + Convolve1DTestParam{2, 2, 2, 2, 1}, + Convolve1DTestParam{161, 1, 1, 10, 1}, + Convolve1DTestParam{900, 1, 1, 10, 1}, + Convolve1DTestParam{640, 3, 3, 128, 1}) + +); + } // namespace } // namespace xla -- GitLab From 690003cc015d6d56630d5836adb6769729bd9c3d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 22 Oct 2017 22:33:18 -0700 Subject: [PATCH 251/573] Add `int64` type `multiples` support for `tf.tile` (#13884) * Add `int64` type `multiples` support for `tf.tile` In the doc of `tf.tile` (tf.tile.__doc__) both `int32` and `int64` are supported for `multiples`. However, the kernel for `int64` is not registered yet. This fix adds the support of `int64` `multiples` so that the behavior matches the description of the docs. Signed-off-by: Yong Tang * Update functors for int64 multiples support in `tf.tile` Signed-off-by: Yong Tang * Update test cases for int64 of multiples in `tf.tile` Signed-off-by: Yong Tang * Add GPU and non GPU tests Signed-off-by: Yong Tang * format with clang-format -i Signed-off-by: Yong Tang * Move Tmultiples after T (as it is auxilliary) And use `use_gpu=True` Signed-off-by: Yong Tang --- tensorflow/core/kernels/tile_functor.h | 39 +-- tensorflow/core/kernels/tile_functor_cpu.cc | 12 +- .../core/kernels/tile_functor_gpu.cu.cc | 12 +- tensorflow/core/kernels/tile_ops.cc | 249 +++++++----------- .../python/kernel_tests/shape_ops_test.py | 18 +- 5 files changed, 149 insertions(+), 181 deletions(-) diff --git a/tensorflow/core/kernels/tile_functor.h b/tensorflow/core/kernels/tile_functor.h index 28af2dace3..189be9239b 100644 --- a/tensorflow/core/kernels/tile_functor.h +++ b/tensorflow/core/kernels/tile_functor.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_KERNELS_TILE_FUNCTOR_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/types.h" @@ -29,13 +30,13 @@ namespace internal { template void TileSimple(const Device& d, Tensor* out, const Tensor& in); -template +template void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice& broadcast_array) { + const gtl::ArraySlice& broadcast_array) { auto x = in.tensor(); auto y = out->tensor(); - Eigen::array b; + Eigen::array b; for (int i = 0; i < NDIM; ++i) b[i] = broadcast_array[i]; if (Eigen::internal::is_same::value) { // Use 32bit indexing to speed up the computations @@ -45,9 +46,9 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, } } -template +template void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice&) { + const gtl::ArraySlice&) { auto x = in.tensor(); auto y = out->tensor(); // In the scalar case we simply copy the input. @@ -58,34 +59,42 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, namespace functor { -template +template struct Tile { void operator()(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice broadcast_array) const { + const gtl::ArraySlice broadcast_array) const { switch (in.dims()) { case 0: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 1: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 2: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 3: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 4: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 5: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 6: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 7: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; default: internal::TileSimple(d, out, in); diff --git a/tensorflow/core/kernels/tile_functor_cpu.cc b/tensorflow/core/kernels/tile_functor_cpu.cc index 5952d49221..b2fd669541 100644 --- a/tensorflow/core/kernels/tile_functor_cpu.cc +++ b/tensorflow/core/kernels/tile_functor_cpu.cc @@ -15,10 +15,10 @@ limitations under the License. #define EIGEN_USE_THREADS +#include "tensorflow/core/kernels/tile_functor.h" #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/kernels/tile_functor.h" namespace tensorflow { @@ -51,7 +51,9 @@ namespace functor { typedef Eigen::ThreadPoolDevice CPUDevice; // Register functors used for Tile functor. -#define DEFINE_TYPE(T) template struct Tile; +#define DEFINE_TYPE(T) \ + template struct Tile; \ + template struct Tile; TF_CALL_bool(DEFINE_TYPE); TF_CALL_float(DEFINE_TYPE); @@ -70,7 +72,9 @@ TF_CALL_string(DEFINE_TYPE); #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#define DEFINE_TYPE(T) template struct Tile; +#define DEFINE_TYPE(T) \ + template struct Tile; \ + template struct Tile; TF_CALL_bool(DEFINE_TYPE); TF_CALL_float(DEFINE_TYPE); @@ -81,7 +85,7 @@ TF_CALL_int16(DEFINE_TYPE); TF_CALL_int64(DEFINE_TYPE); #undef DEFINE_TYPE -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // end namespace functor } // end namespace tensorflow diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.cu.cc index 1c61c3030a..5a36e7567b 100644 --- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc +++ b/tensorflow/core/kernels/tile_functor_gpu.cu.cc @@ -18,10 +18,11 @@ limitations under the License. #define EIGEN_USE_GPU #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/tile_functor.h" + +#include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/kernels/tile_functor.h" #include "tensorflow/core/util/cuda_kernel_helper.h" -#include "tensorflow/core/framework/register_types.h" namespace tensorflow { namespace internal { @@ -60,7 +61,8 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) { host_buf[ndims + i] = out_strides[i]; host_buf[ndims * 2 + i] = in.dim_size(i); } - // Copies the input strides, output strides and input dimension sizes to the device. + // Copies the input strides, output strides and input dimension sizes to the + // device. auto num_bytes = sizeof(int64) * host_buf.size(); auto dev_buf = d.allocate(num_bytes); // NOTE: host_buf is not allocated by CudaHostAllocator, and @@ -84,7 +86,9 @@ namespace functor { typedef Eigen::GpuDevice GPUDevice; // Register functors used for Tile functor. -#define DEFINE_TYPE(T) template struct Tile; +#define DEFINE_TYPE(T) \ + template struct Tile; \ + template struct Tile; TF_CALL_int16(DEFINE_TYPE); TF_CALL_int32(DEFINE_TYPE); diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc index c49ebc0685..4c496a12c2 100644 --- a/tensorflow/core/kernels/tile_ops.cc +++ b/tensorflow/core/kernels/tile_ops.cc @@ -42,14 +42,14 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL // Forward declarations of functors that will be defined in tile_ops_impl.h namespace functor { -template +template struct Tile { void operator()(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice broadcast_array) const; + const gtl::ArraySlice broadcast_array) const; }; template @@ -80,7 +80,7 @@ struct ReduceAndReshape { } // namespace functor // -------------------------------------------------------------------------- -template +template class TileOp : public OpKernel { public: explicit TileOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -105,8 +105,8 @@ class TileOp : public OpKernel { return; } - const gtl::ArraySlice multiples_array(multiples.flat().data(), - input_dims); + const gtl::ArraySlice multiples_array( + multiples.flat().data(), input_dims); TensorShape output_shape; for (int i = 0; i < input_dims; ++i) { OP_REQUIRES( @@ -125,10 +125,10 @@ class TileOp : public OpKernel { // If there's no output, there's nothing to do. if (output_shape.num_elements() == 0) return; -#define HANDLE_TYPE(DT) \ - if (context->input(0).dtype() == DT) { \ - HandleCase
(context, multiples_array, result); \ - return; \ +#define HANDLE_TYPE(DT) \ + if (context->input(0).dtype() == DT) { \ + HandleCase
(context, multiples_array, result); \ + return; \ } #define HANDLE_TYPE_NAME(T) HANDLE_TYPE(DataTypeToEnum::value) @@ -158,27 +158,27 @@ class TileOp : public OpKernel { private: template void HandleCaseImpl(OpKernelContext* context, - const gtl::ArraySlice& multiples_array, + const gtl::ArraySlice& multiples_array, Tensor* result) { typedef typename EnumToDataType
::Type T; - functor::Tile() ( - context->eigen_device(), result, - context->input(0), multiples_array); + functor::Tile()(context->eigen_device(), + result, context->input(0), + multiples_array); } template void HandleCase(OpKernelContext* context, - const gtl::ArraySlice& multiples_array, + const gtl::ArraySlice& multiples_array, Tensor* result); TF_DISALLOW_COPY_AND_ASSIGN(TileOp); }; -template +template template -inline void TileOp::HandleCase( - OpKernelContext* context, const gtl::ArraySlice& multiples_array, - Tensor* result) { +inline void TileOp::HandleCase( + OpKernelContext* context, + const gtl::ArraySlice& multiples_array, Tensor* result) { // TODO(vrv): print out the device name if useful. Currently disabled to avoid // having to use RTTI. LOG(FATAL) << "TileOp: Invalid combination of Device, DT: " @@ -186,25 +186,28 @@ inline void TileOp::HandleCase( << DataTypeString(DT); } -#define HANDLE_CASE(device, dtype) \ - template <> \ - template <> \ - void TileOp::HandleCase( \ - OpKernelContext * context, \ - const gtl::ArraySlice& multiples_array, Tensor* result) { \ - HandleCaseImpl(context, multiples_array, result); \ +#define HANDLE_CASE(device, dtype, Tmultiples) \ + template <> \ + template <> \ + void TileOp::HandleCase( \ + OpKernelContext * context, \ + const gtl::ArraySlice& multiples_array, Tensor* result) { \ + HandleCaseImpl(context, multiples_array, result); \ } -#define HANDLE_TYPE_NAME_CPU(T) \ - HANDLE_CASE(CPUDevice, DataTypeToEnum::value); +#define HANDLE_TYPE_NAME_CPU(T) \ + HANDLE_CASE(CPUDevice, DataTypeToEnum::value, int32); \ + HANDLE_CASE(CPUDevice, DataTypeToEnum::value, int64); -#define HANDLE_TYPE_NAME_GPU(T) \ - HANDLE_CASE(GPUDevice, DataTypeToEnum::value); +#define HANDLE_TYPE_NAME_GPU(T) \ + HANDLE_CASE(GPUDevice, DataTypeToEnum::value, int32); \ + HANDLE_CASE(GPUDevice, DataTypeToEnum::value, int64); #ifdef TENSORFLOW_USE_SYCL -#define HANDLE_TYPE_NAME_SYCL(T) \ - HANDLE_CASE(SYCLDevice, DataTypeToEnum::value); -#endif // TENSORFLOW_USE_SYCL +#define HANDLE_TYPE_NAME_SYCL(T) \ + HANDLE_CASE(SYCLDevice, DataTypeToEnum::value, int32); \ + HANDLE_CASE(SYCLDevice, DataTypeToEnum::value, int64); +#endif // TENSORFLOW_USE_SYCL TF_CALL_bool(HANDLE_TYPE_NAME_CPU); TF_CALL_float(HANDLE_TYPE_NAME_CPU); @@ -235,13 +238,13 @@ TF_CALL_double(HANDLE_TYPE_NAME_SYCL); TF_CALL_int16(HANDLE_TYPE_NAME_SYCL); TF_CALL_int32(HANDLE_TYPE_NAME_SYCL); TF_CALL_int64(HANDLE_TYPE_NAME_SYCL); -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL #undef HANDLE_TYPE_NAME_CPU #undef HANDLE_TYPE_NAME_GPU #ifdef TENSORFLOW_USE_SYCL #undef HANDLE_TYPE_NAME_SYCL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL #undef HANDLE_CASE // -------------------------------------------------------------------------- @@ -494,7 +497,7 @@ TF_CALL_int16(HANDLE_TYPE_NAME_SYCL); TF_CALL_int32(HANDLE_TYPE_NAME_SYCL); TF_CALL_int64(HANDLE_TYPE_NAME_SYCL); #undef HANDLE_TYPE_NAME_SYCL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL #undef HANDLE_TYPE_NAME_CPU #undef HANDLE_TYPE_NAME_GPU @@ -505,127 +508,73 @@ REGISTER_KERNEL_BUILDER(Name("Tile") .Device(DEVICE_CPU) .HostMemory("multiples") .TypeConstraint("Tmultiples"), - TileOp); + TileOp); +REGISTER_KERNEL_BUILDER(Name("Tile") + .Device(DEVICE_CPU) + .HostMemory("multiples") + .TypeConstraint("Tmultiples"), + TileOp); REGISTER_KERNEL_BUILDER( Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"), TileGradientOp); #if GOOGLE_CUDA - -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); - -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); - +#define REGISTER_GPU(type) \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); + +TF_CALL_float(REGISTER_GPU); +TF_CALL_double(REGISTER_GPU); +TF_CALL_half(REGISTER_GPU); +TF_CALL_int16(REGISTER_GPU); +TF_CALL_int32(REGISTER_GPU); +TF_CALL_complex64(REGISTER_GPU); +TF_CALL_complex128(REGISTER_GPU) + +#undef REGISTER_GPU #endif // GOOGLE_CUDA #ifdef TENSORFLOW_USE_SYCL -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); - -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -#endif // TENSORFLOW_USE_SYCL +#define REGISTER_SYCL(type) \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); + + TF_CALL_float(REGISTER_SYCL); +TF_CALL_double(REGISTER_SYCL); + +#undef REGISTER_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py index 52cf904528..a9fc699b21 100644 --- a/tensorflow/python/kernel_tests/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/shape_ops_test.py @@ -411,14 +411,16 @@ class TileTest(test.TestCase): self.assertEqual(7, result) def testSimple(self): - with self.test_session(): - inp = np.random.rand(4, 1).astype(np.float32) - a = constant_op.constant(inp) - tiled = array_ops.tile(a, [1, 4]) - result = tiled.eval() - self.assertEqual(result.shape, (4, 4)) - self.assertEqual([4, 4], tiled.get_shape()) - self.assertTrue((result == np.tile(inp, (1, 4))).all()) + # multiples could be int32 or int64 + for dtype in [dtypes.int32, dtypes.int64]: + with self.test_session(use_gpu=True): + inp = np.random.rand(4, 1).astype(np.float32) + a = constant_op.constant(inp) + tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype)) + result = tiled.eval() + self.assertEqual(result.shape, (4, 4)) + self.assertEqual([4, 4], tiled.get_shape()) + self.assertTrue((result == np.tile(inp, (1, 4))).all()) def testIdentityTileAndGrad(self): with self.test_session(): -- GitLab From 0d437c3beb14c08b5b9c08d806de91d7f3d2c0e3 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 22 Oct 2017 22:50:07 -0700 Subject: [PATCH 252/573] Add int64 padding support for MirrorPad (#13907) * Add int64 padding support for MirrorPad This fix adds int64 padding support for `MirrorPad`. In the `array_ops.cc` the `MirrorPad`/`MirrorPadGrad` has been specified as supporting int64 padding. The related kernels does not have the int64 padding registered though. This fix adds the int64 padding support. This fix also adds additional test cases for coverage. Signed-off-by: Yong Tang * Update template for CPU and GPU support of int64 paddings. Signed-off-by: Yong Tang * Add int64 padding support for MirrorPad Signed-off-by: Yong Tang * Put eigen header first like before, just in case. --- tensorflow/core/kernels/mirror_pad_op.cc | 200 +++++++++++------- tensorflow/core/kernels/mirror_pad_op.h | 13 +- .../core/kernels/mirror_pad_op_cpu_impl.h | 12 +- .../core/kernels/mirror_pad_op_gpu.cu.cc | 32 ++- tensorflow/python/kernel_tests/pad_op_test.py | 19 ++ 5 files changed, 177 insertions(+), 99 deletions(-) diff --git a/tensorflow/core/kernels/mirror_pad_op.cc b/tensorflow/core/kernels/mirror_pad_op.cc index e3643f9447..fbdeaf43eb 100644 --- a/tensorflow/core/kernels/mirror_pad_op.cc +++ b/tensorflow/core/kernels/mirror_pad_op.cc @@ -18,10 +18,10 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/mirror_pad_op.h" - #include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -35,7 +35,7 @@ limitations under the License. namespace tensorflow { -template +template class MirrorPadOp : public OpKernel { public: explicit MirrorPadOp(OpKernelConstruction* context) : OpKernel(context) { @@ -82,10 +82,10 @@ class MirrorPadOp : public OpKernel { // Compute the shape of the output tensor, and allocate it. TensorShape output_shape; - TTypes::ConstMatrix paddings = in1.matrix(); + typename TTypes::ConstMatrix paddings = in1.matrix(); for (int d = 0; d < dims; ++d) { - const int32 before = paddings(d, 0); // Pad before existing elements. - const int32 after = paddings(d, 1); // Pad after existing elements. + const Tpaddings before = paddings(d, 0); // Pad before existing elements. + const Tpaddings after = paddings(d, 1); // Pad after existing elements. OP_REQUIRES(context, before >= 0 && after >= 0, errors::InvalidArgument("paddings must be non-negative: ", before, " ", after)); @@ -121,7 +121,7 @@ class MirrorPadOp : public OpKernel { #define MIRROR_PAD_CASE(i) \ case i: { \ - functor::MirrorPad()( \ + functor::MirrorPad()( \ context->eigen_device(), To32Bit(output->tensor()), \ To32Bit(in0.tensor()), paddings, offset_); \ break; \ @@ -152,20 +152,25 @@ using GpuDevice = Eigen::GpuDevice; namespace functor { // Forward declarations of the functor specializations defined in the sharded // files. -#define DECLARE_CPU_SPEC(T, i) \ - template <> \ - void MirrorPad::operator()( \ - const CpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int); \ - extern template struct MirrorPad; - -#define DECLARE_CPU_SPECS(T) \ - DECLARE_CPU_SPEC(T, 1); \ - DECLARE_CPU_SPEC(T, 2); \ - DECLARE_CPU_SPEC(T, 3); \ - DECLARE_CPU_SPEC(T, 4); \ - DECLARE_CPU_SPEC(T, 5); +#define DECLARE_CPU_SPEC(T, Tpaddings, i) \ + template <> \ + void MirrorPad::operator()( \ + const CpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int); \ + extern template struct MirrorPad; + +#define DECLARE_CPU_SPECS(T) \ + DECLARE_CPU_SPEC(T, int32, 1); \ + DECLARE_CPU_SPEC(T, int32, 2); \ + DECLARE_CPU_SPEC(T, int32, 3); \ + DECLARE_CPU_SPEC(T, int32, 4); \ + DECLARE_CPU_SPEC(T, int32, 5); \ + DECLARE_CPU_SPEC(T, int64, 1); \ + DECLARE_CPU_SPEC(T, int64, 2); \ + DECLARE_CPU_SPEC(T, int64, 3); \ + DECLARE_CPU_SPEC(T, int64, 4); \ + DECLARE_CPU_SPEC(T, int64, 5); TF_CALL_POD_TYPES(DECLARE_CPU_SPECS); @@ -179,7 +184,13 @@ TF_CALL_POD_TYPES(DECLARE_CPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - MirrorPadOp); + MirrorPadOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadOp); // Note that we do register for bool type, but not in the gradient op. TF_CALL_POD_TYPES(REGISTER_KERNEL); @@ -188,20 +199,25 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL); #if GOOGLE_CUDA namespace functor { // Forward declarations of the functor specializations for GPU. -#define DECLARE_GPU_SPEC(T, i) \ - template <> \ - void MirrorPad::operator()( \ - const GpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int); \ - extern template struct MirrorPad; - -#define DECLARE_GPU_SPECS(T) \ - DECLARE_GPU_SPEC(T, 1); \ - DECLARE_GPU_SPEC(T, 2); \ - DECLARE_GPU_SPEC(T, 3); \ - DECLARE_GPU_SPEC(T, 4); \ - DECLARE_GPU_SPEC(T, 5); +#define DECLARE_GPU_SPEC(T, Tpaddings, i) \ + template <> \ + void MirrorPad::operator()( \ + const GpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int); \ + extern template struct MirrorPad; + +#define DECLARE_GPU_SPECS(T) \ + DECLARE_GPU_SPEC(T, int32, 1); \ + DECLARE_GPU_SPEC(T, int32, 2); \ + DECLARE_GPU_SPEC(T, int32, 3); \ + DECLARE_GPU_SPEC(T, int32, 4); \ + DECLARE_GPU_SPEC(T, int32, 5); \ + DECLARE_GPU_SPEC(T, int64, 1); \ + DECLARE_GPU_SPEC(T, int64, 2); \ + DECLARE_GPU_SPEC(T, int64, 3); \ + DECLARE_GPU_SPEC(T, int64, 4); \ + DECLARE_GPU_SPEC(T, int64, 5); TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); #undef DECLARE_GPU_SPECS @@ -215,14 +231,20 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - MirrorPadOp) + MirrorPadOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); #undef REGISTER_GPU_KERNEL #endif // GOOGLE_CUDA // Gradient op. -template +template class MirrorPadGradOp : public OpKernel { public: explicit MirrorPadGradOp(OpKernelConstruction* context) : OpKernel(context) { @@ -269,10 +291,10 @@ class MirrorPadGradOp : public OpKernel { // Compute the shape of the output tensor, and allocate it. TensorShape output_shape; - TTypes::ConstMatrix paddings = in1.matrix(); + typename TTypes::ConstMatrix paddings = in1.matrix(); for (int d = 0; d < dims; ++d) { - const int32 before = paddings(d, 0); // Pad before existing elements. - const int32 after = paddings(d, 1); // Pad after existing elements. + const Tpaddings before = paddings(d, 0); // Pad before existing elements. + const Tpaddings after = paddings(d, 1); // Pad after existing elements. OP_REQUIRES(context, before >= 0 && after >= 0, errors::InvalidArgument("Paddings must be non-negative: ", before, ", ", after)); @@ -308,7 +330,7 @@ class MirrorPadGradOp : public OpKernel { #define MIRROR_PAD_GRAD_CASE(k) \ case k: { \ - functor::MirrorPadGrad()( \ + functor::MirrorPadGrad()( \ context->eigen_device(), To32Bit(output->tensor()), \ To32Bit(in0.tensor()), paddings, offset_, \ To32Bit(scratch.tensor())); \ @@ -337,33 +359,45 @@ class MirrorPadGradOp : public OpKernel { namespace functor { // Forward declarations of the functor specializations defined in the sharded // files. -#define DECLARE_CPU_SPEC(T, k) \ - template <> \ - void MirrorPadGrad::operator()( \ - const CpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int, typename TTypes::Tensor); \ - extern template struct MirrorPadGrad; - -#define DECLARE_CPU_SPECS(T) \ - DECLARE_CPU_SPEC(T, 1); \ - DECLARE_CPU_SPEC(T, 2); \ - DECLARE_CPU_SPEC(T, 3); \ - DECLARE_CPU_SPEC(T, 4); \ - DECLARE_CPU_SPEC(T, 5); +#define DECLARE_CPU_SPEC(T, Tpaddings, k) \ + template <> \ + void MirrorPadGrad::operator()( \ + const CpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int, \ + typename TTypes::Tensor); \ + extern template struct MirrorPadGrad; + +#define DECLARE_CPU_SPECS(T) \ + DECLARE_CPU_SPEC(T, int32, 1); \ + DECLARE_CPU_SPEC(T, int32, 2); \ + DECLARE_CPU_SPEC(T, int32, 3); \ + DECLARE_CPU_SPEC(T, int32, 4); \ + DECLARE_CPU_SPEC(T, int32, 5); \ + DECLARE_CPU_SPEC(T, int64, 1); \ + DECLARE_CPU_SPEC(T, int64, 2); \ + DECLARE_CPU_SPEC(T, int64, 3); \ + DECLARE_CPU_SPEC(T, int64, 4); \ + DECLARE_CPU_SPEC(T, int64, 5); TF_CALL_NUMBER_TYPES(DECLARE_CPU_SPECS); #undef DECLARE_CPU_SPECS #undef DECLARE_CPU_SPEC } // namespace functor -#define REGISTER_KERNEL(type) \ - REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("paddings"), \ - MirrorPadGradOp); +#define REGISTER_KERNEL(type) \ + REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadGradOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadGradOp); TF_CALL_NUMBER_TYPES(REGISTER_KERNEL); #undef REGISTER_KERNEL @@ -371,20 +405,26 @@ TF_CALL_NUMBER_TYPES(REGISTER_KERNEL); #if GOOGLE_CUDA namespace functor { // Forward declarations of the functor specializations for GPU. -#define DECLARE_GPU_SPEC(T, k) \ - template <> \ - void MirrorPadGrad::operator()( \ - const GpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int, typename TTypes::Tensor); \ - extern template struct MirrorPadGrad; - -#define DECLARE_GPU_SPECS(T) \ - DECLARE_GPU_SPEC(T, 1); \ - DECLARE_GPU_SPEC(T, 2); \ - DECLARE_GPU_SPEC(T, 3); \ - DECLARE_GPU_SPEC(T, 4); \ - DECLARE_GPU_SPEC(T, 5); +#define DECLARE_GPU_SPEC(T, Tpaddings, k) \ + template <> \ + void MirrorPadGrad::operator()( \ + const GpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int, \ + typename TTypes::Tensor); \ + extern template struct MirrorPadGrad; + +#define DECLARE_GPU_SPECS(T) \ + DECLARE_GPU_SPEC(T, int32, 1); \ + DECLARE_GPU_SPEC(T, int32, 2); \ + DECLARE_GPU_SPEC(T, int32, 3); \ + DECLARE_GPU_SPEC(T, int32, 4); \ + DECLARE_GPU_SPEC(T, int32, 5); \ + DECLARE_GPU_SPEC(T, int64, 1); \ + DECLARE_GPU_SPEC(T, int64, 2); \ + DECLARE_GPU_SPEC(T, int64, 3); \ + DECLARE_GPU_SPEC(T, int64, 4); \ + DECLARE_GPU_SPEC(T, int64, 5); TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); #undef DECLARE_GPU_SPECS @@ -398,7 +438,13 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - MirrorPadGradOp) + MirrorPadGradOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadGradOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); #undef REGISTER_GPU_KERNEL diff --git a/tensorflow/core/kernels/mirror_pad_op.h b/tensorflow/core/kernels/mirror_pad_op.h index b83d2223d0..81150a9e79 100644 --- a/tensorflow/core/kernels/mirror_pad_op.h +++ b/tensorflow/core/kernels/mirror_pad_op.h @@ -64,9 +64,8 @@ class TensorMirrorPadOp StorageKind; typedef typename Eigen::internal::traits::Index Index; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - TensorMirrorPadOp(const XprType& expr, const PaddingDimensions& padding_dims, - Index offset) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorMirrorPadOp( + const XprType& expr, const PaddingDimensions& padding_dims, Index offset) : xpr_(expr), padding_dims_(padding_dims), offset_(offset) {} EIGEN_DEVICE_FUNC @@ -336,12 +335,12 @@ namespace functor { // offset argument must be either 0 or 1. This controls whether the boundary // values are replicated (offset == 0) or not replicated (offset == 1). -template +template struct MirrorPad { void operator()(const Device& device, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - TTypes::ConstMatrix padding, int offset) { + typename TTypes::ConstMatrix padding, int offset) { Eigen::array, Dims> padding_dims; for (int i = 0; i < Dims; ++i) { @@ -363,12 +362,12 @@ struct MirrorPad { // offset argument must be either 0 or 1. This controls whether the boundary // values are replicated (offset == 0) or not replicated (offset == 1). -template +template struct MirrorPadGrad { void operator()(const Device& device, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - TTypes::ConstMatrix paddings, int offset, + typename TTypes::ConstMatrix paddings, int offset, typename TTypes::Tensor scratch) { // Copy the gradient input into the scratch buffer. scratch.device(device) = input; diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h index 9864f5633a..bb22b2aa91 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h +++ b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h @@ -25,13 +25,17 @@ namespace tensorflow { using CpuDevice = Eigen::ThreadPoolDevice; -#define DEFINE_CPU_SPECS(T) \ - template struct functor::MirrorPad; +#define DEFINE_CPU_SPECS(T) \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; TF_CALL_POD_TYPES(DEFINE_CPU_SPECS); #undef DEFINE_CPU_SPECS -#define DEFINE_CPU_SPECS(T) \ - template struct functor::MirrorPadGrad; +#define DEFINE_CPU_SPECS(T) \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; TF_CALL_NUMBER_TYPES(DEFINE_CPU_SPECS); #undef DEFINE_CPU_SPECS diff --git a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc index 8074aa9624..dbd0a9bd8f 100644 --- a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc +++ b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc @@ -25,17 +25,27 @@ namespace tensorflow { using GpuDevice = Eigen::GpuDevice; -#define DEFINE_GPU_SPECS(T) \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; +#define DEFINE_GPU_SPECS(T) \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS); #undef DEFINE_GPU_SPECS diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py index 1af43e6067..2c766e3640 100644 --- a/tensorflow/python/kernel_tests/pad_op_test.py +++ b/tensorflow/python/kernel_tests/pad_op_test.py @@ -193,6 +193,25 @@ class PadOpTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Unknown padding mode"): array_ops.pad(x, [[1, 0], [2, 1]], mode="weird").eval() + def testPaddingTypes(self): + paddings = [[1, 0], [2, 3], [0, 2]] + inputs = np.random.randint(-100, 100, (4, 4, 3)).astype(np.float32) + for mode in ("CONSTANT", "REFLECT", "SYMMETRIC", "reflect", "symmetric", + "constant"): + for padding_dtype in [dtypes.int32, dtypes.int64]: + np_val = self._npPad(inputs, + paddings, + mode=mode, + constant_values=0) + with self.test_session(use_gpu=True): + tf_val = array_ops.pad(inputs, + constant_op.constant(paddings, padding_dtype), + mode=mode, + constant_values=0) + out = tf_val.eval() + self.assertAllEqual(np_val, out) + self.assertShapeEqual(np_val, tf_val) + def testIntTypes(self): # TODO(touts): Figure out why the padding tests do not work on GPU # for int types and rank > 2. -- GitLab From ac0004e71120e237989422bd4a7441df72613072 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 22 Oct 2017 22:50:20 -0700 Subject: [PATCH 253/573] Add int64 shape support on GPU for stateless random ops. (#13908) * Add int64 shape support on GPU for stateless random ops. This fix adds int64 shape support on GPU for stateless random ops `StatelessRandomUniform`, `StatelessRandomNormal`, `StatelessTruncatedNormal`. The int64 shape for stateless random ops is already supported on CPU with int32/int64 processed properly through `MakeShape`. However, on GPU a type constraint `.TypeConstraint("T")` has been improperly added. Such a type constraint actually prevents an int64 shape type to run on GPU. (As a comparision, no type constraint on CPU). This fix removes the type constraint and allows int64 shape to be run on GPU. This fix also adds test cases for int64 shape support on stateless random ops. Signed-off-by: Yong Tang * Add test cases for int64 shape support for stateless random ops. Signed-off-by: Yong Tang * Add int32 to shape types tested. --- .../kernel_tests/stateless_random_ops_test.py | 16 ++++++++++++++++ tensorflow/core/kernels/stateless_random_ops.cc | 3 --- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py index 9a36bdc2f9..cd4d46aa07 100644 --- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py +++ b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.contrib import stateless +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops @@ -79,6 +80,21 @@ class StatelessOpsTest(test.TestCase): for s1, v1 in values: self.assertEqual(s0 == s1, np.all(v0 == v1)) + def testShapeType(self): + with self.test_session(use_gpu=True): + for shape_dtype in [dtypes.int32, dtypes.int64]: + seed_t = array_ops.placeholder(dtypes.int64, shape=[2]) + seeds = [(x, y) for x in range(5) for y in range(5)] * 3 + for stateless_op, _ in CASES: + for shape in (), (3,), (2, 5): + pure = stateless_op(constant_op.constant(shape, dtype=shape_dtype), + seed=seed_t) + values = [(seed, pure.eval(feed_dict={seed_t: seed})) + for seed in seeds] + for s0, v0 in values: + for s1, v1 in values: + self.assertEqual(s0 == s1, np.all(v0 == v1)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc index 79d0c07acd..f6fb0a121d 100644 --- a/tensorflow/core/kernels/stateless_random_ops.cc +++ b/tensorflow/core/kernels/stateless_random_ops.cc @@ -137,7 +137,6 @@ TF_CALL_double(REGISTER); .Device(DEVICE_GPU) \ .HostMemory("shape") \ .HostMemory("seed") \ - .TypeConstraint("T") \ .TypeConstraint("dtype"), \ StatelessRandomOp >); \ @@ -146,7 +145,6 @@ TF_CALL_double(REGISTER); .Device(DEVICE_GPU) \ .HostMemory("shape") \ .HostMemory("seed") \ - .TypeConstraint("T") \ .TypeConstraint("dtype"), \ StatelessRandomOp >); \ @@ -155,7 +153,6 @@ TF_CALL_double(REGISTER); .Device(DEVICE_GPU) \ .HostMemory("shape") \ .HostMemory("seed") \ - .TypeConstraint("T") \ .TypeConstraint("dtype"), \ StatelessRandomOp< \ GPUDevice, \ -- GitLab From 9b9cbbe2a69b7fcec72d82f271cb90839c3035b7 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 22 Oct 2017 23:02:28 -0700 Subject: [PATCH 254/573] Add int64 Tperm type support for `Transpose` (#13909) * Add int64 Tperm type support for `Transpose` This fix adds int64 Tperm support for `Transpose`. In `array_ops.cc`, `Transpose` and `ConjugateTranspose` have been specified as accepting int32 and int64 perm types. However, only int32 kernels has been registered. This fix adds the int64 perm support by removing the constraint on Tperm, resolve the type at runtime, and copying the data type accordingly to correctly handle the int64/int32 types. Additional tests have been added as well. Signed-off-by: Yong Tang * Add test cases for int64 of perm in Transpose. Signed-off-by: Yong Tang * Add namespace to hide PermutationHelper Signed-off-by: Yong Tang * Enable use_gpu=True for perm type test. Signed-off-by: Yong Tang * extra // namespace annotation * Adding a comment about int32 casting that should be safe. Permutations only contain values that refer to dimensions, and the maximum number of dimensions we have is 254, so an int32 is always safe here. --- tensorflow/core/kernels/transpose_op.cc | 134 ++++++++++-------- .../python/kernel_tests/transpose_op_test.py | 13 ++ 2 files changed, 85 insertions(+), 62 deletions(-) diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc index e151b38d90..20f0edf309 100644 --- a/tensorflow/core/kernels/transpose_op.cc +++ b/tensorflow/core/kernels/transpose_op.cc @@ -91,6 +91,26 @@ REGISTER_KERNEL_BUILDER(Name("InvertPermutation") InvertPermutationOp); #endif // TENSORFLOW_USE_SYCL +namespace { +template +Status PermutationHelper(const Tensor& perm, const int dims, + std::vector* permutation) { + auto Vperm = perm.vec(); + if (dims != Vperm.size()) { + return errors::InvalidArgument("transpose expects a vector of size ", dims, + ". But input(1) is a vector of size ", + Vperm.size()); + } + // using volatile instead of SubtleMustCopy here so that the + // asynchrony boundary is permutation. + const volatile Tperm* perm_begin = + reinterpret_cast(Vperm.data()); + *permutation = std::vector(perm_begin, perm_begin + dims); + + return Status::OK(); +} +} // namespace + // output = TransposeOp(T input, T perm) takes a tensor // of type T and rank N, and a permutation of 0, 1, ..., N-1. It // shuffles the dimensions of the input tensor according to permutation. @@ -113,17 +133,16 @@ void TransposeOp::Compute(OpKernelContext* ctx) { OP_REQUIRES(ctx, TensorShapeUtils::IsVector(perm.shape()), errors::InvalidArgument("perm must be a vector, not ", perm.shape().DebugString())); - auto Vperm = perm.vec(); + + // Although Tperm may be an int64 type, an int32 is sufficient to hold + // dimension range values, so the narrowing here should be safe. + std::vector permutation; const int dims = input.dims(); - OP_REQUIRES(ctx, dims == Vperm.size(), - errors::InvalidArgument( - "transpose expects a vector of size ", input.dims(), - ". But input(1) is a vector of size ", Vperm.size())); - // using volatile instead of SubtleMustCopy here so that the - // asynchrony boundary is permutation. - const volatile int32* perm_begin = - reinterpret_cast(Vperm.data()); - const std::vector permutation(perm_begin, perm_begin + dims); + if (perm.dtype() == DT_INT32) { + OP_REQUIRES_OK(ctx, PermutationHelper(perm, dims, &permutation)); + } else { + OP_REQUIRES_OK(ctx, PermutationHelper(perm, dims, &permutation)); + } TensorShape shape; // Check whether permutation is a permutation of integers of [0 .. dims). @@ -142,10 +161,9 @@ void TransposeOp::Compute(OpKernelContext* ctx) { } } for (int i = 0; i < dims; ++i) { - OP_REQUIRES( - ctx, bits[i], - errors::InvalidArgument(i, " is missing from {", - str_util::Join(permutation, ","), "}.")); + OP_REQUIRES(ctx, bits[i], errors::InvalidArgument( + i, " is missing from {", + str_util::Join(permutation, ","), "}.")); } // 0-D, 1-D, and identity transposes do nothing. @@ -185,18 +203,16 @@ Status ConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx, } #ifdef INTEL_MKL -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - MklTransposeCpuOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + MklTransposeCpuOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ MklConjugateTransposeCpuOp); TF_CALL_ALL_TYPES(REGISTER); REGISTER(bfloat16); @@ -204,18 +220,16 @@ REGISTER(bfloat16); #else // INTEL_MKL -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - TransposeCpuOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + TransposeCpuOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ ConjugateTransposeCpuOp); TF_CALL_ALL_TYPES(REGISTER) REGISTER(bfloat16); @@ -238,18 +252,16 @@ Status ConjugateTransposeGpuOp::DoTranspose(OpKernelContext* ctx, perm, out); } -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - TransposeGpuOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + TransposeGpuOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ ConjugateTransposeGpuOp); TF_CALL_POD_TYPES(REGISTER); #undef REGISTER @@ -270,18 +282,16 @@ Status ConjugateTransposeSyclOp::DoTranspose(OpKernelContext* ctx, return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), in, perm, out); } -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - TransposeSyclOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + TransposeSyclOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ ConjugateTransposeSyclOp); TF_CALL_POD_TYPES(REGISTER); #undef REGISTER diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py index 3b352937c8..c551d9c3d0 100644 --- a/tensorflow/python/kernel_tests/transpose_op_test.py +++ b/tensorflow/python/kernel_tests/transpose_op_test.py @@ -317,6 +317,19 @@ class TransposeTest(test.TestCase): np.arange(0, 8).reshape([2, 4]).astype(np.float32), np.array([1, 0]).astype(np.int32)) + def testPermType(self): + for perm_dtype in [np.int64, np.int32]: + x = np.arange(0, 8).reshape([2, 4]).astype(np.float32) + p = np.array([1, 0]).astype(perm_dtype) + np_ans = np.copy(x).transpose(p) + with self.test_session(use_gpu=True): + inx = ops.convert_to_tensor(x) + inp = constant_op.constant(p) + y = array_ops.transpose(inx, inp) + tf_ans = y.eval() + self.assertShapeEqual(np_ans, y) + self.assertAllEqual(np_ans, tf_ans) + def testHalf(self): self._compare(np.arange(0, 21).reshape([3, 7]).astype(np.float16)) self._compare(np.arange(0, 210).reshape([2, 3, 5, 7]).astype(np.float16)) -- GitLab From eea089bdb66597c9e66180d39b94eea2c17be93e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 06:00:06 -0700 Subject: [PATCH 255/573] K-FAC: Multi-tower support for ConvDiagonalFB. PiperOrigin-RevId: 173105412 --- .../python/kernel_tests/fisher_blocks_test.py | 202 +++++++++++++++++- .../kernel_tests/layer_collection_test.py | 6 +- .../contrib/kfac/python/ops/fisher_blocks.py | 70 ++++-- .../kfac/python/ops/layer_collection.py | 27 ++- 4 files changed, 271 insertions(+), 34 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py index 9b13756e62..80855da2e9 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py @@ -328,17 +328,11 @@ class FullyConnectedDiagonalFB(test.TestCase): multiply_result: Result of FisherBlock.multiply(params) multiply_inverse_result: Result of FisherBlock.multiply_inverse(params) """ - - def _as_tensors(tensor_or_tuple): - if isinstance(tensor_or_tuple, (tuple, list)): - return tuple(ops.convert_to_tensor(t) for t in tensor_or_tuple) - return ops.convert_to_tensor(tensor_or_tuple) - with ops.Graph().as_default(), self.test_session() as sess: - inputs = [_as_tensors(i) for i in inputs] - outputs = [_as_tensors(o) for o in outputs] - output_grads = [_as_tensors(og) for og in output_grads] - params = _as_tensors(params) + inputs = as_tensors(inputs) + outputs = as_tensors(outputs) + output_grads = as_tensors(output_grads) + params = as_tensors(params) block = fb.FullyConnectedDiagonalFB( lc.LayerCollection(), has_bias=isinstance(params, (tuple, list))) @@ -464,6 +458,188 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +class ConvDiagonalFBTest(test.TestCase): + + def setUp(self): + super(ConvDiagonalFBTest, self).setUp() + + self.batch_size = 2 + self.height = 8 + self.width = 4 + self.input_channels = 6 + self.output_channels = 3 + self.kernel_size = 1 + + self.inputs = np.random.randn(self.batch_size, self.height, self.width, + self.input_channels).astype(np.float32) + self.outputs = np.zeros( + [self.batch_size, self.height, self.width, + self.output_channels]).astype(np.float32) + self.output_grads = np.random.randn( + self.batch_size, self.height, self.width, self.output_channels).astype( + np.float32) + self.w = np.random.randn(self.kernel_size, self.kernel_size, + self.input_channels, self.output_channels).astype( + np.float32) + self.b = np.random.randn(self.output_channels).astype(np.float32) + + def fisherApprox(self, has_bias=False): + """Fisher approximation using default inputs.""" + if has_bias: + inputs = np.concatenate( + [self.inputs, + np.ones([self.batch_size, self.height, self.width, 1])], + axis=-1) + else: + inputs = self.inputs + return self.buildDiagonalFisherApproximation(inputs, self.output_grads, + self.kernel_size) + + def buildDiagonalFisherApproximation(self, inputs, output_grads, kernel_size): + r"""Builds explicit diagonal Fisher approximation. + + Fisher's diagonal is (d loss / d w)'s elements squared for + d/dw = E[\sum_{loc} outer(input_{loc}, output_grad_{loc})] + + where the expectation is taken over examples and the sum over (x, y) + locations upon which the convolution is applied. + + Args: + inputs: np.array of shape [batch_size, height, width, input_channels]. + output_grads: np.array of shape [batch_size, height, width, + output_channels]. + kernel_size: int. height and width of kernel. + + Returns: + Diagonal np.array of shape [num_params, num_params] for num_params = + kernel_size^2 * input_channels * output_channels. + """ + batch_size, height, width, input_channels = inputs.shape + assert output_grads.shape[0] == batch_size + assert output_grads.shape[1] == height + assert output_grads.shape[2] == width + output_channels = output_grads.shape[3] + + # If kernel_size == 1, then we don't need to worry about capturing context + # around the pixel upon which a convolution is applied. This makes testing + # easier. + assert kernel_size == 1, "kernel_size != 1 isn't supported." + num_locations = height * width + inputs = np.reshape(inputs, [batch_size, num_locations, input_channels]) + output_grads = np.reshape(output_grads, + [batch_size, num_locations, output_channels]) + + fisher_diag = np.zeros((input_channels, output_channels)) + for i in range(batch_size): + # Each example's approximation is a square(sum-of-outer-products). + example_fisher_diag = np.zeros((input_channels, output_channels)) + for j in range(num_locations): + example_fisher_diag += np.outer(inputs[i, j], output_grads[i, j]) + fisher_diag += np.square(example_fisher_diag) + + # Normalize by batch_size (not num_locations). + return np.diag(fisher_diag.flatten()) / batch_size + + def testMultiply(self): + result, _ = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct Fisher-vector product. + expected_result = self.fisherApprox().dot(self.w.flatten()) + expected_result = expected_result.reshape([ + self.kernel_size, self.kernel_size, self.input_channels, + self.output_channels + ]) + + self.assertAllClose(expected_result, result) + + def testMultiplyInverse(self): + _, result = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct inverse Fisher-vector product. + expected_result = np.linalg.inv(self.fisherApprox()).dot(self.w.flatten()) + expected_result = expected_result.reshape([ + self.kernel_size, self.kernel_size, self.input_channels, + self.output_channels + ]) + + self.assertAllClose(expected_result, result, atol=1e-3) + + def testRegisterAdditionalMinibatch(self): + """Ensure 1 big minibatch and 2 small minibatches are equivalent.""" + multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps( + self.w, [self.inputs], [self.outputs], [self.output_grads]) + multiply_result_small, multiply_inverse_result_small = ( + self.runFisherBlockOps(self.w, + np.split(self.inputs, 2), + np.split(self.outputs, 2), + np.split(self.output_grads, 2))) + + self.assertAllClose(multiply_result_big, multiply_result_small) + self.assertAllClose(multiply_inverse_result_big, + multiply_inverse_result_small) + + def testMultiplyHasBias(self): + result, _ = self.runFisherBlockOps((self.w, self.b), [self.inputs], + [self.outputs], [self.output_grads]) + # Clone 'b' along 'input_channels' dimension. + b_filter = np.tile( + np.reshape(self.b, [1, 1, 1, self.output_channels]), + [self.kernel_size, self.kernel_size, 1, 1]) + params = np.concatenate([self.w, b_filter], axis=2) + expected_result = self.fisherApprox(True).dot(params.flatten()) + + # Extract 'b' from concatenated parameters. + expected_result = expected_result.reshape([ + self.kernel_size, self.kernel_size, self.input_channels + 1, + self.output_channels + ]) + expected_result = (expected_result[:, :, 0:-1, :], np.reshape( + expected_result[:, :, -1, :], [self.output_channels])) + + self.assertEqual(len(result), 2) + self.assertAllClose(expected_result[0], result[0]) + self.assertAllClose(expected_result[1], result[1]) + + def runFisherBlockOps(self, params, inputs, outputs, output_grads): + """Run Ops guaranteed by FisherBlock interface. + + Args: + params: Tensor or 2-tuple of Tensors. Represents weights or weights and + bias of this layer. + inputs: list of Tensors of shape [batch_size, input_size]. Inputs to + layer. + outputs: list of Tensors of shape [batch_size, output_size]. + Preactivations produced by layer. + output_grads: list of Tensors of shape [batch_size, output_size]. + Gradient of loss with respect to 'outputs'. + + Returns: + multiply_result: Result of FisherBlock.multiply(params) + multiply_inverse_result: Result of FisherBlock.multiply_inverse(params) + """ + with ops.Graph().as_default(), self.test_session() as sess: + inputs = as_tensors(inputs) + outputs = as_tensors(outputs) + output_grads = as_tensors(output_grads) + params = as_tensors(params) + + block = fb.ConvDiagonalFB( + lc.LayerCollection(), params, strides=[1, 1, 1, 1], padding='SAME') + for (i, o) in zip(inputs, outputs): + block.register_additional_minibatch(i, o) + + block.instantiate_factors((output_grads,), damping=0.0) + + sess.run(tf_variables.global_variables_initializer()) + sess.run(block._factor.make_covariance_update_op(0.0)) + multiply_result = sess.run(block.multiply(params)) + multiply_inverse_result = sess.run(block.multiply_inverse(params)) + + return multiply_result, multiply_inverse_result + + class ConvKFCBasicFBTest(test.TestCase): def _testConvKFCBasicFBInitParams(self, params): @@ -583,5 +759,11 @@ class ConvKFCBasicFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +def as_tensors(tensor_or_tuple): + """Converts a potentially nested tuple of np.array to Tensors.""" + if isinstance(tensor_or_tuple, (tuple, list)): + return tuple(as_tensors(t) for t in tensor_or_tuple) + return ops.convert_to_tensor(tensor_or_tuple) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 53d40da586..b444e87170 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -89,6 +89,10 @@ class LayerCollectionTest(test.TestCase): lc.register_conv2d( array_ops.constant(4), [1, 1, 1, 1], 'SAME', array_ops.ones((1, 1, 1, 1)), array_ops.constant(3)) + lc.register_conv2d( + array_ops.constant(4), [1, 1, 1, 1], 'SAME', + array_ops.ones((1, 1, 1, 1)), array_ops.constant(3), + approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_generic( array_ops.constant(5), 16, approx=layer_collection.APPROX_FULL_NAME) lc.register_generic( @@ -96,7 +100,7 @@ class LayerCollectionTest(test.TestCase): 16, approx=layer_collection.APPROX_DIAGONAL_NAME) - self.assertEqual(5, len(lc.get_blocks())) + self.assertEqual(6, len(lc.get_blocks())) def testRegisterBlocksMultipleRegistrations(self): with ops.Graph().as_default(): diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 6cca2272d7..5e822b5fe3 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -227,7 +227,7 @@ class FullyConnectedDiagonalFB(FisherBlock): 'w'. For an example 'x' that produces layer inputs 'a' and output preactivations 's', - v(x, y, w) = vec( x (d loss / d s)^T ) + v(x, y, w) = vec( a (d loss / d s)^T ) This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding to the layer's parameters 'w'. @@ -309,13 +309,29 @@ class FullyConnectedDiagonalFB(FisherBlock): class ConvDiagonalFB(FisherBlock): """FisherBlock for convolutional layers using a diagonal approx. - Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator. - """ + Estimates the Fisher Information matrix's diagonal entries for a convolutional + layer. Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" + estimator. - # TODO(jamesmartens): add units tests for this class + Let 'params' be a vector parameterizing a model and 'i' an arbitrary index + into it. We are interested in Fisher(params)[i, i]. This is, - def __init__(self, layer_collection, params, inputs, outputs, strides, - padding): + Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] + = E[ v(x, y, params)[i] ^ 2 ] + + Consider a convoluational layer in this model with (unshared) filter matrix + 'w'. For an example image 'x' that produces layer inputs 'a' and output + preactivations 's', + + v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T ) + + where 'loc' is a single (x, y) location in an image. + + This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding + to the layer's parameters 'w'. + """ + + def __init__(self, layer_collection, params, strides, padding): """Creates a ConvDiagonalFB block. Args: @@ -325,37 +341,39 @@ class ConvDiagonalFB(FisherBlock): kernel alone, a Tensor of shape [kernel_height, kernel_width, in_channels, out_channels]. If kernel and bias, a tuple of 2 elements containing the previous and a Tensor of shape [out_channels]. - inputs: A Tensor of shape [batch_size, height, width, in_channels]. - Input activations to this layer. - outputs: A Tensor of shape [batch_size, height, width, out_channels]. - Output pre-activations from this layer. strides: The stride size in this layer (1-D Tensor of length 4). - padding: The padding in this layer (1-D of Tensor length 4). + padding: The padding in this layer (e.g. "SAME"). """ - self._inputs = inputs - self._outputs = outputs - self._strides = strides + self._inputs = [] + self._outputs = [] + self._strides = tuple(strides) if isinstance(strides, list) else strides self._padding = padding self._has_bias = isinstance(params, (tuple, list)) fltr = params[0] if self._has_bias else params self._filter_shape = tuple(fltr.shape.as_list()) - input_shape = tuple(inputs.shape.as_list()) - self._num_locations = ( - input_shape[1] * input_shape[2] // (strides[1] * strides[2])) - super(ConvDiagonalFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): + # Concatenate inputs, grads_list into single Tensors. + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + + # Infer number of locations upon which convolution is applied. + inputs_shape = tuple(inputs.shape.as_list()) + self._num_locations = ( + inputs_shape[1] * inputs_shape[2] // + (self._strides[1] * self._strides[2])) + if NORMALIZE_DAMPING_POWER: damping /= self._num_locations**NORMALIZE_DAMPING_POWER self._damping = damping self._factor = self._layer_collection.make_or_get_factor( fisher_factors.ConvDiagonalFactor, - (self._inputs, grads_list, self._filter_shape, self._strides, - self._padding, self._has_bias)) + (inputs, grads_list, self._filter_shape, self._strides, self._padding, + self._has_bias)) def multiply_inverse(self, vector): reshaped_vect = utils.layer_params_to_mat2d(vector) @@ -370,6 +388,18 @@ class ConvDiagonalFB(FisherBlock): def tensors_to_compute_grads(self): return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, height, width, input_size]. Inputs to + the convolution. + outputs: Tensor of shape [batch_size, height, width, output_size]. Layer + preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + class KroneckerProductFB(FisherBlock): """A base class for FisherBlocks with separate input and output factors. diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index beb8ef136e..10ef554351 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -273,9 +273,9 @@ class LayerCollection(object): fb.ConvKFCBasicFB(self, params, inputs, outputs, strides, padding)) elif approx == APPROX_DIAGONAL_NAME: - self.register_block(params, - fb.ConvDiagonalFB(self, params, inputs, outputs, - strides, padding)) + block = fb.ConvDiagonalFB(self, params, strides, padding) + block.register_additional_minibatch(inputs, outputs) + self.register_block(params, block) def register_generic(self, params, batch_size, approx=APPROX_DIAGONAL_NAME): params = params if isinstance(params, (tuple, list)) else (params,) @@ -379,6 +379,27 @@ class LayerCollection(object): self._loss_dict[name] = loss def make_or_get_factor(self, cls, args): + """Insert 'cls(args)' into 'self.fisher_factors' if not already present. + + Wraps constructor in 'tf.variable_scope()' to ensure variables constructed + in 'cls.__init__' are placed under this LayerCollection's scope. + + Args: + cls: Class that implements FisherFactor. + args: Tuple of arguments to pass into 'cls's constructor. Must be + hashable. + + Returns: + Instance of 'cls' found in self.fisher_factors. + """ + try: + hash(args) + except TypeError: + raise TypeError(( + "Unable to use (cls, args) = ({}, {}) as a key in " + "LayerCollection.fisher_factors. The pair cannot be hashed." + ).format(cls, args)) + with variable_scope.variable_scope(self._var_scope): return utils.setdefault(self.fisher_factors, (cls, args), lambda: cls(*args)) -- GitLab From dc13a8e2f7cfd56121347f5596f8b5a770da41c9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 06:03:28 -0700 Subject: [PATCH 256/573] Fix import of meta graphs with partitioned variables into a scope. Saver inspects SliceInfo to decide the variable name when creating a checkpoint. Before this fix even if a partitioned variable ("weights") was imported into a scope "a" it would still be checkpointed as ("weights") instead of ("a/weights") since import_scoped_meta_graph was not adjusting the SliceInfo. WARNING: if you use import_meta_graph on graphs with partitioned_variables WITH an import_scope argument AND then create a Saver to write/read checkpoints this change may break your checkpoint loading. PiperOrigin-RevId: 173105796 --- .../python/framework/meta_graph_test.py | 39 +++++++++++++++++++ tensorflow/python/ops/variables.py | 3 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py index 65abb69599..06cee46bf6 100644 --- a/tensorflow/python/framework/meta_graph_test.py +++ b/tensorflow/python/framework/meta_graph_test.py @@ -36,8 +36,10 @@ from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import random_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -657,5 +659,42 @@ class MetaGraphWithVariableScopeTest(test.TestCase): initializer = variables.local_variables_initializer() +class ExportImportAcrossScopesTest(test.TestCase): + + def testPartionedVariables(self): + def make_graph_with_partitioned_variables(): + variable_scope.get_variable( + name="weights", + partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0), + initializer=random_ops.truncated_normal([100, 10])) + self._testExportImportAcrossScopes(make_graph_with_partitioned_variables) + + def _testExportImportAcrossScopes(self, graph_fn): + """Tests export and importing a graph across scopes. + + Args: + graph_fn: A closure that creates a graph on the current scope. + """ + with ops.Graph().as_default() as original_graph: + with variable_scope.variable_scope("dropA/dropB/keepA"): + graph_fn() + exported_meta_graph_def = meta_graph.export_scoped_meta_graph( + graph=original_graph, + export_scope="dropA/dropB")[0] + + with ops.Graph().as_default() as imported_graph: + meta_graph.import_scoped_meta_graph( + exported_meta_graph_def, + import_scope="importA") + + with ops.Graph().as_default() as expected_graph: + with variable_scope.variable_scope("importA/keepA"): + graph_fn() + + result = meta_graph.export_scoped_meta_graph(graph=imported_graph)[0] + expected = meta_graph.export_scoped_meta_graph(graph=expected_graph)[0] + self.assertProtoEquals(expected, result) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 90b4f25d81..0272f77176 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -394,7 +394,8 @@ class Variable(object): import_scope=import_scope)) if variable_def.HasField("save_slice_info_def"): self._save_slice_info = Variable.SaveSliceInfo( - save_slice_info_def=variable_def.save_slice_info_def) + save_slice_info_def=variable_def.save_slice_info_def, + import_scope=import_scope) else: self._save_slice_info = None self._caching_device = None -- GitLab From 670dddf4ad81c67fc76b370bf7b9d77263824358 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 06:53:02 -0700 Subject: [PATCH 257/573] Multi-minibatch support for tf.contrib.kfac.fisher_blocks.FullyConnectedKFACBasicFB. PiperOrigin-RevId: 173109677 --- .../python/kernel_tests/fisher_blocks_test.py | 41 ++++++++-------- .../contrib/kfac/python/ops/fisher_blocks.py | 48 ++++++++++++++----- .../contrib/kfac/python/ops/fisher_factors.py | 8 ++++ .../kfac/python/ops/layer_collection.py | 6 +-- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py index 80855da2e9..85ac08a1eb 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py @@ -356,50 +356,51 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): random_seed.set_random_seed(200) inputs = array_ops.constant([1., 2.]) outputs = array_ops.constant([3., 4.]) - block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), inputs, - outputs) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection()) + block.register_additional_minibatch(inputs, outputs) - self.assertAllEqual(outputs, block.tensors_to_compute_grads()) + self.assertAllEqual([outputs], block.tensors_to_compute_grads()) def testInstantiateFactorsHasBias(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2.], [3., 4.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=True) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=True) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) def testInstantiateFactorsNoBias(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2.], [3., 4.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) def testMultiplyInverseTuple(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) - vector = (np.arange(2, 6).reshape(2, 2).astype(np.float32), np.arange( - 1, 3).reshape(2, 1).astype(np.float32)) + vector = ( + np.arange(2, 6).reshape(2, 2).astype(np.float32), # + np.arange(1, 3).reshape(2, 1).astype(np.float32)) output = block.multiply_inverse((array_ops.constant(vector[0]), array_ops.constant(vector[1]))) @@ -413,10 +414,10 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2.], [3., 4.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -436,11 +437,11 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): inputs = array_ops.zeros([32, input_dim]) outputs = array_ops.zeros([32, output_dim]) params = array_ops.zeros([input_dim, output_dim]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 damping = 0. # This test is only valid without damping. - block.instantiate_factors((grads,), damping) + block.instantiate_factors(([grads],), damping) sess.run(state_ops.assign(block._input_factor._cov, _make_psd(3))) sess.run(state_ops.assign(block._output_factor._cov, _make_psd(2))) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 5e822b5fe3..754c2cc853 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -367,7 +367,7 @@ class ConvDiagonalFB(FisherBlock): (self._strides[1] * self._strides[2])) if NORMALIZE_DAMPING_POWER: - damping /= self._num_locations**NORMALIZE_DAMPING_POWER + damping /= self._num_locations ** NORMALIZE_DAMPING_POWER self._damping = damping self._factor = self._layer_collection.make_or_get_factor( @@ -478,34 +478,60 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB): K-FAC paper (https://arxiv.org/abs/1503.05671) """ - def __init__(self, layer_collection, inputs, outputs, has_bias=False): + def __init__(self, layer_collection, has_bias=False): """Creates a FullyConnectedKFACBasicFB block. Args: layer_collection: The collection of all layers in the K-FAC approximate Fisher information matrix to which this FisherBlock belongs. - inputs: The Tensor of input activations to this layer. - outputs: The Tensor of output pre-activations from this layer. has_bias: Whether the component Kronecker factors have an additive bias. (Default: False) """ - self._inputs = inputs - self._outputs = outputs + self._inputs = [] + self._outputs = [] self._has_bias = has_bias super(FullyConnectedKFACBasicFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): - self._input_factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedKroneckerFactor, - ((self._inputs,), self._has_bias)) - self._output_factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedKroneckerFactor, (grads_list,)) + """Instantiate Kronecker Factors for this FisherBlock. + + Args: + grads_list: List of list of Tensors. grads_list[i][j] is the + gradient of the loss with respect to 'outputs' from source 'i' and + tower 'j'. Each Tensor has shape [tower_minibatch_size, output_size]. + damping: 0-D Tensor or float. 'damping' * identity is approximately added + to this FisherBlock's Fisher approximation. + """ + # TODO(b/68033310): Validate which of, + # (1) summing on a single device (as below), or + # (2) on each device in isolation and aggregating + # is faster. + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + + self._input_factor = self._layer_collection.make_or_get_factor( # + fisher_factors.FullyConnectedKroneckerFactor, # + ((inputs,), self._has_bias)) + self._output_factor = self._layer_collection.make_or_get_factor( # + fisher_factors.FullyConnectedKroneckerFactor, # + (grads_list,)) self._register_damped_input_and_output_inverses(damping) def tensors_to_compute_grads(self): return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, input_size]. Inputs to the + matrix-multiply. + outputs: Tensor of shape [batch_size, output_size]. Layer preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + class ConvKFCBasicFB(KroneckerProductFB): """FisherBlock for 2D convolutional layers using the basic KFC approx. diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index 86a1782fcf..b8b524406c 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -573,6 +573,14 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor): """ def __init__(self, tensors, has_bias=False): + """Instantiate FullyConnectedKroneckerFactor. + + Args: + tensors: List of Tensors of shape [batch_size, n]. Represents either a + layer's inputs or its output's gradients. + has_bias: bool. If True, assume this factor is for the layer's inputs and + append '1' to each row. + """ # The tensor argument is either a tensor of input activations or a tensor of # output pre-activation gradients. self._has_bias = has_bias diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 10ef554351..ceb1131f28 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -255,9 +255,9 @@ class LayerCollection(object): approx=APPROX_KRONECKER_NAME): has_bias = isinstance(params, (tuple, list)) if approx == APPROX_KRONECKER_NAME: - self.register_block(params, - fb.FullyConnectedKFACBasicFB(self, inputs, outputs, - has_bias)) + block = fb.FullyConnectedKFACBasicFB(self, has_bias) + block.register_additional_minibatch(inputs, outputs) + self.register_block(params, block) elif approx == APPROX_DIAGONAL_NAME: block = fb.FullyConnectedDiagonalFB(self, has_bias) block.register_additional_minibatch(inputs, outputs) -- GitLab From 434695921de7cfd713b789533173e1e0c3fc7691 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 08:00:39 -0700 Subject: [PATCH 258/573] K-FAC: _check_registration() supports multiple towers. PiperOrigin-RevId: 173115870 --- .../kernel_tests/layer_collection_test.py | 14 ++++++-- tensorflow/contrib/kfac/python/ops/BUILD | 2 ++ .../contrib/kfac/python/ops/fisher_blocks.py | 34 +++++++++++++++++++ .../kfac/python/ops/layer_collection.py | 10 +++--- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index b444e87170..1da811dc0a 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -313,10 +313,20 @@ class LayerCollectionTest(test.TestCase): self.assertTrue(all([var.name.startswith(scope) for var in variables])) def testGetUseCountMap(self): + """Ensure get_use_count_map() sums 'num_registered_minibatches'.""" + + class MockFisherBlock(object): + + num_registered_minibatches = 2 + lc = layer_collection.LayerCollection() - lc.fisher_blocks = {'a': 1, ('a', 'c'): 2, ('b', 'c'): 2} + lc.fisher_blocks = { + 'a': MockFisherBlock(), + ('a', 'c'): MockFisherBlock(), + ('b', 'c'): MockFisherBlock() + } use_count_map = lc.get_use_count_map() - self.assertDictEqual({'a': 2, 'b': 1, 'c': 2}, use_count_map) + self.assertDictEqual({'a': 4, 'b': 2, 'c': 4}, use_count_map) if __name__ == '__main__': diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD index 8b82f6e314..5d5046c9ec 100644 --- a/tensorflow/contrib/kfac/python/ops/BUILD +++ b/tensorflow/contrib/kfac/python/ops/BUILD @@ -113,7 +113,9 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 754c2cc853..7ef755c35e 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -114,6 +114,14 @@ class FisherBlock(object): """ pass + @abc.abstractproperty + def num_registered_minibatches(self): + """Number of minibatches registered for this FisherBlock. + + Typically equal to the number of towers in a multi-tower setup. + """ + pass + class FullFB(FisherBlock): """FisherBlock using a full matrix estimate (no approximations). @@ -164,6 +172,10 @@ class FullFB(FisherBlock): def tensors_to_compute_grads(self): return self._params + @property + def num_registered_minibatches(self): + return 1 # Multiple minibatches not supported. + class NaiveDiagonalFB(FisherBlock): """FisherBlock using a diagonal matrix approximation. @@ -209,6 +221,10 @@ class NaiveDiagonalFB(FisherBlock): def tensors_to_compute_grads(self): return self._params + @property + def num_registered_minibatches(self): + return 1 # Multiple minibatches not supported. + class FullyConnectedDiagonalFB(FisherBlock): """FisherBlock for fully-connected (dense) layers using a diagonal approx. @@ -305,6 +321,12 @@ class FullyConnectedDiagonalFB(FisherBlock): self._inputs.append(inputs) self._outputs.append(outputs) + @property + def num_registered_minibatches(self): + result = len(self._inputs) + assert result == len(self._outputs) + return result + class ConvDiagonalFB(FisherBlock): """FisherBlock for convolutional layers using a diagonal approx. @@ -400,6 +422,10 @@ class ConvDiagonalFB(FisherBlock): self._inputs.append(inputs) self._outputs.append(outputs) + @property + def num_registered_minibatches(self): + return len(self._inputs) + class KroneckerProductFB(FisherBlock): """A base class for FisherBlocks with separate input and output factors. @@ -532,6 +558,10 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB): self._inputs.append(inputs) self._outputs.append(outputs) + @property + def num_registered_minibatches(self): + return 1 # Multiple minibatches not supported. + class ConvKFCBasicFB(KroneckerProductFB): """FisherBlock for 2D convolutional layers using the basic KFC approx. @@ -591,6 +621,10 @@ class ConvKFCBasicFB(KroneckerProductFB): def tensors_to_compute_grads(self): return self._outputs + @property + def num_registered_minibatches(self): + return 1 # Multiple minibatches not supported. + def _concat_along_batch_dim(tensor_list): """Concatenate tensors along batch (first) dimension. diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index ceb1131f28..49279954dc 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -27,6 +27,8 @@ from __future__ import print_function from collections import defaultdict from collections import OrderedDict +import six + from tensorflow.contrib.kfac.python.ops import fisher_blocks as fb from tensorflow.contrib.kfac.python.ops import loss_functions as lf from tensorflow.contrib.kfac.python.ops import utils @@ -82,8 +84,8 @@ class LayerParametersDict(OrderedDict): return key -# TODO(duckworthd): add capability for LayerCollection to be "finalized" -# and do this when it gets used by FisherEstimator / KfacOptimizer +# TODO(b/68034464): add capability for LayerCollection to be "finalized" +# and do this when it gets used by FisherEstimator / KfacOptimizer. class LayerCollection(object): @@ -211,10 +213,10 @@ class LayerCollection(object): def get_use_count_map(self): """Returns a dict of variables to their number of registrations.""" vars_to_uses = defaultdict(int) - for key in self.fisher_blocks.keys(): + for key, block in six.iteritems(self.fisher_blocks): key = key if isinstance(key, (tuple, list)) else (key,) for k in key: - vars_to_uses[k] += 1 + vars_to_uses[k] += block.num_registered_minibatches return vars_to_uses def get_blocks(self): -- GitLab From ed03c433ae5c89075717476f9bc1e731de656c6f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 23 Oct 2017 15:26:02 +0000 Subject: [PATCH 259/573] Add int64 type `multiples` support for TileGrad. This fix is a follow up of #13884 to add int64 type of `multiples` support for TileGrad for completeness. Signed-off-by: Yong Tang --- tensorflow/core/kernels/tile_ops.cc | 89 +++++++++++++++++++---------- 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc index 4c496a12c2..fa5afe6a31 100644 --- a/tensorflow/core/kernels/tile_ops.cc +++ b/tensorflow/core/kernels/tile_ops.cc @@ -248,7 +248,7 @@ TF_CALL_int64(HANDLE_TYPE_NAME_SYCL); #undef HANDLE_CASE // -------------------------------------------------------------------------- -template +template class TileGradientOp : public OpKernel { public: explicit TileGradientOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -273,10 +273,10 @@ class TileGradientOp : public OpKernel { return; } - const gtl::ArraySlice multiples_array(multiples.flat().data(), - input_dims); + const gtl::ArraySlice multiples_array( + multiples.flat().data(), input_dims); TensorShape output_shape; - std::vector input_dim_size_vec; + std::vector input_dim_size_vec; for (int i = 0; i < input_dims; ++i) { OP_REQUIRES( context, multiples_array[i] > 0, @@ -337,19 +337,19 @@ class TileGradientOp : public OpKernel { private: template void HandleCase(OpKernelContext* context, - const std::vector& input_dims, - const gtl::ArraySlice& multiples_array, + const std::vector& input_dims, + const gtl::ArraySlice& multiples_array, Tensor* result); template void HandleCaseImpl(OpKernelContext* context, - const std::vector& input_dims, - const gtl::ArraySlice& multiples_array, + const std::vector& input_dims, + const gtl::ArraySlice& multiples_array, Tensor* result) { typedef typename EnumToDataType
::Type T; bool reduction_only = true; - std::vector reduction_dims; + std::vector reduction_dims; for (int i = 0; i < NDIM; ++i) { if (input_dims[i] > multiples_array[i] && multiples_array[i] > 1) { @@ -411,7 +411,8 @@ class TileGradientOp : public OpKernel { template void HandleReduce(OpKernelContext* context, - const std::vector& reduce_dim_in, Tensor* result) { + const std::vector& reduce_dim_in, + Tensor* result) { static_assert(NDIM >= REDUCENDIM, "Too many reduced dimensions"); Eigen::DSizes reduce_dim; Eigen::DSizes reshape_dim; @@ -432,34 +433,41 @@ class TileGradientOp : public OpKernel { TF_DISALLOW_COPY_AND_ASSIGN(TileGradientOp); }; -template +template template -inline void TileGradientOp::HandleCase( - OpKernelContext* context, const std::vector& input_dims, - const gtl::ArraySlice& multiples_array, Tensor* result) { +inline void TileGradientOp::HandleCase( + OpKernelContext* context, const std::vector& input_dims, + const gtl::ArraySlice& multiples_array, Tensor* result) { LOG(FATAL) << "TileGradientOp: Invalid combination of Device, DT and NDIM: " << MakeTypeIndex().name() << ", " << DataTypeString(DT) << ", " << NDIM; } -#define HANDLE_CASE(device, T, dtype, ndim) \ +#define HANDLE_CASE(device, T, dtype, Tmultiples, ndim) \ template <> \ template <> \ - void TileGradientOp::HandleCase( \ - OpKernelContext * context, const std::vector& input_dims, \ - const gtl::ArraySlice& multiples_array, Tensor* result) { \ + void TileGradientOp::HandleCase( \ + OpKernelContext * context, const std::vector& input_dims, \ + const gtl::ArraySlice& multiples_array, Tensor* result) { \ HandleCaseImpl(context, input_dims, multiples_array, result); \ } // 0-D handled specially above -#define HANDLE_CASE_DIM(device, T, dtype) \ - HANDLE_CASE(device, T, dtype, 1); \ - HANDLE_CASE(device, T, dtype, 2); \ - HANDLE_CASE(device, T, dtype, 3); \ - HANDLE_CASE(device, T, dtype, 4); \ - HANDLE_CASE(device, T, dtype, 5); \ - HANDLE_CASE(device, T, dtype, 6); \ - HANDLE_CASE(device, T, dtype, 7); +#define HANDLE_CASE_DIM(device, T, dtype) \ + HANDLE_CASE(device, T, dtype, int32, 1); \ + HANDLE_CASE(device, T, dtype, int32, 2); \ + HANDLE_CASE(device, T, dtype, int32, 3); \ + HANDLE_CASE(device, T, dtype, int32, 4); \ + HANDLE_CASE(device, T, dtype, int32, 5); \ + HANDLE_CASE(device, T, dtype, int32, 6); \ + HANDLE_CASE(device, T, dtype, int32, 7); \ + HANDLE_CASE(device, T, dtype, int64, 1); \ + HANDLE_CASE(device, T, dtype, int64, 2); \ + HANDLE_CASE(device, T, dtype, int64, 3); \ + HANDLE_CASE(device, T, dtype, int64, 4); \ + HANDLE_CASE(device, T, dtype, int64, 5); \ + HANDLE_CASE(device, T, dtype, int64, 6); \ + HANDLE_CASE(device, T, dtype, int64, 7); #define HANDLE_TYPE_NAME_CPU(T) \ HANDLE_CASE_DIM(CPUDevice, T, DataTypeToEnum::value); @@ -514,9 +522,16 @@ REGISTER_KERNEL_BUILDER(Name("Tile") .HostMemory("multiples") .TypeConstraint("Tmultiples"), TileOp); -REGISTER_KERNEL_BUILDER( - Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"), - TileGradientOp); +REGISTER_KERNEL_BUILDER(Name("TileGrad") + .Device(DEVICE_CPU) + .HostMemory("multiples") + .TypeConstraint("Tmultiples"), + TileGradientOp); +REGISTER_KERNEL_BUILDER(Name("TileGrad") + .Device(DEVICE_CPU) + .HostMemory("multiples") + .TypeConstraint("Tmultiples"), + TileGradientOp); #if GOOGLE_CUDA #define REGISTER_GPU(type) \ @@ -537,7 +552,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("T") \ .TypeConstraint("Tmultiples") \ .HostMemory("multiples"), \ - TileGradientOp); + TileGradientOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); TF_CALL_float(REGISTER_GPU); TF_CALL_double(REGISTER_GPU); @@ -569,7 +590,13 @@ TF_CALL_complex128(REGISTER_GPU) .TypeConstraint("T") \ .TypeConstraint("Tmultiples") \ .HostMemory("multiples"), \ - TileGradientOp); + TileGradientOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); TF_CALL_float(REGISTER_SYCL); TF_CALL_double(REGISTER_SYCL); -- GitLab From 2845bfcd64cea4405135b3c7034e9aa28896dff4 Mon Sep 17 00:00:00 2001 From: Tim Harley Date: Mon, 23 Oct 2017 08:54:57 -0700 Subject: [PATCH 260/573] Avoid listing all modified Enter/RefEnter nodes on INFO, use VLOG(1) instead. Leave a single, simple, message on INFO. PiperOrigin-RevId: 173121726 --- tensorflow/core/debug/debug_graph_utils.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/debug/debug_graph_utils.cc b/tensorflow/core/debug/debug_graph_utils.cc index 2559808b59..4539ea5c0c 100644 --- a/tensorflow/core/debug/debug_graph_utils.cc +++ b/tensorflow/core/debug/debug_graph_utils.cc @@ -221,21 +221,26 @@ Status DebugNodeInserter::InsertNodes( } void DebugNodeInserter::DeparallelizeWhileLoops(Graph* graph, Device* device) { + bool deparallelized_a_loop = false; for (Node* node : graph->nodes()) { if (node->IsEnter()) { const AttrValue* parallel_iterations = node->attrs().Find("parallel_iterations"); if (parallel_iterations && parallel_iterations->i() > 1) { - LOG(INFO) << "For debugging, tfdbg is changing the " - << "parallel_iterations attribute of the Enter/RefEnter " - << "node \"" << node->name() << "\" on device \"" - << device->name() << "\" from " << parallel_iterations->i() - << " to 1. (This does not affect subsequent non-debug " - << "runs.)"; + deparallelized_a_loop = true; + VLOG(1) << "Changing the parallel_iterations attribute of the " + << "Enter/RefEnter node \"" << node->name() << "\" on device \"" + << device->name() << "\" from " << parallel_iterations->i() + << " to 1."; node->AddAttr("parallel_iterations", 1); } } } + if (deparallelized_a_loop) { + LOG(INFO) << "For debugging, tfdbg has set the parallel_iterations " + << "attribute of all scheduled Enter/RefEnter nodes to 1. (This " + << "does not affect subsequent non-debug runs.)"; + } } // static -- GitLab From 4f7503a876e20e6d58c9aec3f44214b98bcfdbbb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 08:55:00 -0700 Subject: [PATCH 261/573] K-FAC: Support for registering multiple minibatches with register_fully_connected() PiperOrigin-RevId: 173121735 --- .../kernel_tests/layer_collection_test.py | 67 +++++++++++++++++++ .../kfac/python/ops/layer_collection.py | 64 +++++++++++++++--- .../kfac/python/ops/layer_collection_lib.py | 1 + 3 files changed, 122 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 1da811dc0a..432937d803 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -282,6 +282,73 @@ class LayerCollectionTest(test.TestCase): single_loss = sess.run(lc.total_loss()) self.assertAlmostEqual(7.6983433, single_loss) + def testRegisterFullyConnectedReuse(self): + """Ensure the 'reuse' keyword argument function as intended.""" + with ops.Graph().as_default(): + inputs = [ + array_ops.ones([2, 10]), # + array_ops.zeros([5, 10]) + ] + outputs = [ + array_ops.zeros([2, 5]), # + array_ops.ones([5, 5]) + ] + params = ( + variable_scope.get_variable('w', [10, 5]), # + variable_scope.get_variable('b', [5])) + + # Fails on second if reuse=False. + lc = layer_collection.LayerCollection() + lc.register_fully_connected(params, inputs[0], outputs[0]) + with self.assertRaises(ValueError): + lc.register_fully_connected(params, inputs[1], outputs[1], reuse=False) + + # Succeeds on second if reuse=True. + lc = layer_collection.LayerCollection() + lc.register_fully_connected(params, inputs[0], outputs[0]) + lc.register_fully_connected(params, inputs[1], outputs[1], reuse=True) + + # Fails on second if reuse=VARIABLE_SCOPE and no variable reuse. + lc = layer_collection.LayerCollection() + lc.register_fully_connected(params, inputs[0], outputs[0]) + with self.assertRaises(ValueError): + lc.register_fully_connected( + params, + inputs[1], + outputs[1], + reuse=layer_collection.VARIABLE_SCOPE) + + # Succeeds on second if reuse=VARIABLE_SCOPE and variable reuse. + lc = layer_collection.LayerCollection() + lc.register_fully_connected(params, inputs[0], outputs[0]) + with variable_scope.variable_scope( + variable_scope.get_variable_scope(), reuse=True): + lc.register_fully_connected( + params, + inputs[1], + outputs[1], + reuse=layer_collection.VARIABLE_SCOPE) + + # Fails if block type changes. + lc = layer_collection.LayerCollection() + lc.register_fully_connected( + params, + inputs[0], + outputs[0], + approx=layer_collection.APPROX_KRONECKER_NAME) + with self.assertRaises(ValueError): + lc.register_fully_connected( + params, + inputs[1], + outputs[1], + approx=layer_collection.APPROX_DIAGONAL_NAME, + reuse=True) + + # Fails if reuse requested but no FisherBlock exists. + lc = layer_collection.LayerCollection() + with self.assertRaises(KeyError): + lc.register_fully_connected(params, inputs[0], outputs[0], reuse=True) + def testMakeOrGetFactor(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 49279954dc..cd711d0561 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -39,10 +39,15 @@ from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import nest +# Names for various approximations that can be requested for Fisher blocks. APPROX_KRONECKER_NAME = "kron" APPROX_DIAGONAL_NAME = "diagonal" APPROX_FULL_NAME = "full" +# Possible value for 'reuse' keyword argument. Sets 'reuse' to +# tf.get_variable_scope().reuse. +VARIABLE_SCOPE = "VARIABLE_SCOPE" + # TODO(jamesmartens): need to add find_canonical_output back into this somewhere @@ -254,18 +259,57 @@ class LayerCollection(object): params, inputs, outputs, - approx=APPROX_KRONECKER_NAME): + approx=APPROX_KRONECKER_NAME, + reuse=VARIABLE_SCOPE): + """Registers a fully connnected layer. + + Args: + params: Tensor or 2-tuple of Tensors corresponding to weight and bias of + this layer. Weight matrix should have shape [input_size, output_size]. + Bias should have shape [output_size]. + inputs: Tensor of shape [batch_size, input_size]. Inputs to layer. + outputs: Tensor of shape [batch_size, output_size]. Preactivations + produced by layer. + approx: str. One of APPROX_KRONECKER_NAME or APPROX_DIAGONAL_NAME. + reuse: bool or str. If True, reuse an existing FisherBlock. If False, + create a new FisherBlock. If VARIABLE_SCOPE, use + tf.get_variable_scope().reuse. + + Raises: + ValueError: For improper value to 'approx'. + KeyError: If reuse == True but no FisherBlock found for 'params'. + ValueError: If reuse == True and FisherBlock found but of the wrong type. + """ + approx_to_block_types = { + APPROX_KRONECKER_NAME: fb.FullyConnectedKFACBasicFB, + APPROX_DIAGONAL_NAME: fb.FullyConnectedDiagonalFB, + } + + if approx not in approx_to_block_types: + raise ValueError("Bad value {} for approx.".format(approx)) + + block_type = approx_to_block_types[approx] has_bias = isinstance(params, (tuple, list)) - if approx == APPROX_KRONECKER_NAME: - block = fb.FullyConnectedKFACBasicFB(self, has_bias) - block.register_additional_minibatch(inputs, outputs) - self.register_block(params, block) - elif approx == APPROX_DIAGONAL_NAME: - block = fb.FullyConnectedDiagonalFB(self, has_bias) - block.register_additional_minibatch(inputs, outputs) - self.register_block(params, block) + + if reuse == VARIABLE_SCOPE: + reuse = variable_scope.get_variable_scope().reuse + + if reuse: + block = self.fisher_blocks.get(params, None) + if block is None: + raise KeyError( + "Reuse requested but no FisherBlock found for params {}.".format( + params)) + if not isinstance(block, block_type): + raise ValueError( + "Requested block of type {} but block of type {} already exists " + "for params {}.".format(block_type, type(block), params)) + else: - raise ValueError("Bad value {} for approx.".format(approx)) + block = block_type(self, has_bias) + self.register_block(params, block) + + block.register_additional_minibatch(inputs, outputs) def register_conv2d(self, params, strides, padding, inputs, outputs, approx=APPROX_KRONECKER_NAME): diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py b/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py index 63a9b173bc..d6bf61a210 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py @@ -35,6 +35,7 @@ _allowed_symbols = [ "APPROX_KRONECKER_NAME", "APPROX_DIAGONAL_NAME", "APPROX_FULL_NAME", + "VARIABLE_SCOPE", ] remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) -- GitLab From fc56349b7f1afec33c88358a06dab12dda5736a4 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Mon, 23 Oct 2017 09:34:30 -0700 Subject: [PATCH 262/573] [tf.data] Convert dataset arguments to tensors as early as possible. This change raises a `TypeError` earlier if (for example) the `batch_size` argument to `Dataset.batch()` has the incorrect type. PiperOrigin-RevId: 173126678 --- tensorflow/python/data/ops/dataset_ops.py | 29 +++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 5f2e6296a8..151556994f 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -1057,21 +1057,21 @@ class RangeDataset(Dataset): def _parse_args(self, *args): if len(args) == 1: self._start = self._build_tensor(0, "start") - self._stop = args[0] + self._stop = self._build_tensor(args[0], "stop") self._step = self._build_tensor(1, "step") elif len(args) == 2: - self._start = args[0] - self._stop = args[1] + self._start = self._build_tensor(args[0], "start") + self._stop = self._build_tensor(args[1], "stop") self._step = self._build_tensor(1, "step") elif len(args) == 3: - self._start = args[0] - self._stop = args[1] - self._step = args[2] + self._start = self._build_tensor(args[0], "start") + self._stop = self._build_tensor(args[1], "stop") + self._step = self._build_tensor(args[2], "step") else: raise ValueError("Invalid arguments to RangeDataset: %s" % str(args)) def _build_tensor(self, int64_value, name): - return constant_op.constant(int64_value, dtype=dtypes.int64, name=name) + return ops.convert_to_tensor(int64_value, dtype=dtypes.int64, name=name) def _as_variant_tensor(self): return gen_dataset_ops.range_dataset( @@ -1217,7 +1217,8 @@ class BatchDataset(Dataset): """See `Dataset.batch()` for details.""" super(BatchDataset, self).__init__() self._input_dataset = input_dataset - self._batch_size = batch_size + self._batch_size = ops.convert_to_tensor(batch_size, dtype=dtypes.int64, + name="batch_size") def _as_variant_tensor(self): return gen_dataset_ops.batch_dataset( @@ -1285,7 +1286,8 @@ class PaddedBatchDataset(Dataset): """See `Dataset.batch()` for details.""" super(PaddedBatchDataset, self).__init__() self._input_dataset = input_dataset - self._batch_size = batch_size + self._batch_size = ops.convert_to_tensor(batch_size, dtype=dtypes.int64, + name="batch_size") padding_values = (padding_values if padding_values is not None else self._default_padding(input_dataset)) self._padded_shapes = nest.map_structure_up_to( @@ -1509,8 +1511,10 @@ class InterleaveDataset(Dataset): self._map_func = tf_map_func self._map_func.add_to_graph(ops.get_default_graph()) - self._cycle_length = ops.convert_to_tensor(cycle_length, dtype=dtypes.int64) - self._block_length = ops.convert_to_tensor(block_length, dtype=dtypes.int64) + self._cycle_length = ops.convert_to_tensor(cycle_length, dtype=dtypes.int64, + name="cycle_length") + self._block_length = ops.convert_to_tensor(block_length, dtype=dtypes.int64, + name="block_length") def _as_variant_tensor(self): return gen_dataset_ops.interleave_dataset( @@ -1587,7 +1591,8 @@ class PrefetchDataset(Dataset): """See `Dataset.prefetch()` for details.""" super(PrefetchDataset, self).__init__() self._input_dataset = input_dataset - self._buffer_size = ops.convert_to_tensor(buffer_size, dtype=dtypes.int64) + self._buffer_size = ops.convert_to_tensor(buffer_size, dtype=dtypes.int64, + name="buffer_size") def _as_variant_tensor(self): return gen_dataset_ops.prefetch_dataset( -- GitLab From 46ab25e4de87bc6873697e28f747fb8ae3579755 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Mon, 23 Oct 2017 09:36:32 -0700 Subject: [PATCH 263/573] [XLA] Add support for convolutions with no spatial dimensions PiperOrigin-RevId: 173126950 --- .../xla/client/computation_builder.cc | 2 +- .../xla/service/algebraic_simplifier.cc | 27 ++++++++++++------- .../xla/service/algebraic_simplifier.h | 11 +++++--- .../compiler/xla/service/hlo_cost_analysis.cc | 4 ++- .../compiler/xla/service/hlo_evaluator.cc | 2 +- .../compiler/xla/service/shape_inference.cc | 10 ++----- 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc index dcbdb3525e..b9977fb2f8 100644 --- a/tensorflow/compiler/xla/client/computation_builder.cc +++ b/tensorflow/compiler/xla/client/computation_builder.cc @@ -663,7 +663,7 @@ bool ComputationBuilder::VerifyConvolution( return false; } int num_dims = ShapeUtil::Rank(lhs_shape); - if (num_dims < 3) { + if (num_dims < 2) { NoteError(InvalidArgument( "Convolution expects argument arrays with >= 3 dimensions. " "Got: %s and %s", diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 39e8430ed3..8b3886cc7a 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -201,17 +201,18 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { static bool Run( HloComputation* computation, bool is_layout_sensitive, AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification); + bool enable_dot_simplification, bool enable_conv_simplification); private: explicit AlgebraicSimplifierVisitor( HloComputation* computation, bool is_layout_sensitive, AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification) + bool enable_dot_simplification, bool enable_conv_simplification) : computation_(computation), is_layout_sensitive_(is_layout_sensitive), valid_bitcast_callback_(std::move(valid_bitcast_callback)), - enable_dot_simplification_(enable_dot_simplification) {} + enable_dot_simplification_(enable_dot_simplification), + enable_conv_simplification_(enable_conv_simplification) {} // Convenience method for replacing an instruction with a bitcast. void ReplaceWithBitcast(HloInstruction* instruction); @@ -287,15 +288,18 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { // Disable dot simplication on platforms where it causes a slowdown. bool enable_dot_simplification_; + + // Disable convolution simplication on platforms where it causes a slowdown. + bool enable_conv_simplification_; }; bool AlgebraicSimplifierVisitor::Run( HloComputation* computation, bool is_layout_sensitive, AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification) { - AlgebraicSimplifierVisitor visitor(computation, is_layout_sensitive, - std::move(valid_bitcast_callback), - enable_dot_simplification); + bool enable_dot_simplification, bool enable_conv_simplification) { + AlgebraicSimplifierVisitor visitor( + computation, is_layout_sensitive, std::move(valid_bitcast_callback), + enable_dot_simplification, enable_conv_simplification); TF_CHECK_OK(computation->Accept(&visitor)); return visitor.changed_; } @@ -1459,6 +1463,9 @@ Status AlgebraicSimplifierVisitor::HandleTranspose(HloInstruction* transpose) { Status AlgebraicSimplifierVisitor::HandleConvolution( HloInstruction* convolution, HloInstruction* lhs, HloInstruction* rhs, const Window& window) { + if (!enable_conv_simplification_) { + return Status::OK(); + } // HandleConvolution tries to replace a convolution with a DOT instruction. // // Only add when bitcasts can be used: @@ -1962,9 +1969,9 @@ StatusOr AlgebraicSimplifier::Run(HloModule* module) { "AlgebraicSimplifier::Run(), before:\n" + module->ToString()); bool changed = false; for (auto* comp : module->MakeNonfusionComputations()) { - if (AlgebraicSimplifierVisitor::Run(comp, is_layout_sensitive_, - valid_bitcast_callback_, - enable_dot_simplification_)) { + if (AlgebraicSimplifierVisitor::Run( + comp, is_layout_sensitive_, valid_bitcast_callback_, + enable_dot_simplification_, enable_conv_simplification_)) { changed = true; } } diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h index 4295a3227a..a9f476178c 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.h +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h @@ -40,11 +40,13 @@ class AlgebraicSimplifier : public HloPassInterface { // bitcasts. AlgebraicSimplifier(bool is_layout_sensitive, ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification = true) + bool enable_dot_simplification = true, + bool enable_conv_simplification = true) : is_layout_sensitive_(is_layout_sensitive), valid_bitcast_callback_(std::move(valid_bitcast_callback)), - enable_dot_simplification_(enable_dot_simplification) {} - ~AlgebraicSimplifier() override {} + enable_dot_simplification_(enable_dot_simplification), + enable_conv_simplification_(enable_conv_simplification) {} + ~AlgebraicSimplifier() override = default; tensorflow::StringPiece name() const override { return "algsimp"; } // Run algebraic simplification on the given computation. Returns whether the @@ -57,6 +59,9 @@ class AlgebraicSimplifier : public HloPassInterface { // Enable dot simplication on platforms where it is profitable. bool enable_dot_simplification_; + + // Enable convolution simplication on platforms where it is profitable. + bool enable_conv_simplification_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc index 84d55d4b5f..ca99fd6de8 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc @@ -398,7 +398,9 @@ Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution, // For each output element, we do one fma per element in the kernel at some // given output feature index. const int64 fmas_per_output_element = - ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features; + output_features > 0 + ? ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features + : 0; const int64 output_elements = ShapeUtil::ElementsIn(convolution->shape()); current_properties_[kFlopsKey] = output_elements * fmas_per_output_element * kFmaFlops; diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index 5fd891835d..e8f88427da 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -547,7 +547,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { const auto& dnums = conv->convolution_dimension_numbers(); const int64 num_spatial_dims = dnums.spatial_dimensions_size(); CHECK_EQ(num_spatial_dims, dnums.kernel_spatial_dimensions_size()); - CHECK_GE(num_spatial_dims, 1); + CHECK_GE(num_spatial_dims, 0); CHECK_EQ(window.dimensions_size(), num_spatial_dims); const auto lhs_rank = ShapeUtil::Rank(lhs_shape); diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index 6be6b77e85..1df1022442 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -1385,14 +1385,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( "Window: %s", window.DebugString().c_str()); } - int num_spatial_dims = dnums.spatial_dimensions_size(); - if (num_spatial_dims < 1) { - return InvalidArgument( - "Convolution requires at least one spatial dimension.\n" - "Window: %s", - window.DebugString().c_str()); - } + const int num_spatial_dims = dnums.spatial_dimensions_size(); if (window.dimensions_size() != num_spatial_dims) { return InvalidArgument( "Window must have same number of dimensions as dimension numbers.\n" @@ -1400,7 +1394,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( window.DebugString().c_str(), dnums.DebugString().c_str()); } - int num_dims = num_spatial_dims + 2; + const int num_dims = num_spatial_dims + 2; if (ShapeUtil::Rank(lhs) != num_dims) { return InvalidArgument( "The LHS argument to a convolution should have rank %d.\n" -- GitLab From 03b02ffc9e542a7f40d98debd711e537f7f3bb04 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 23 Oct 2017 09:44:24 -0700 Subject: [PATCH 264/573] Put Bazel mirror URLs first PiperOrigin-RevId: 173127955 --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 8ba8748aae..02540bd843 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -173,8 +173,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "mkl_dnn", urls = [ - "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", + "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", ], sha256 = "0d529ad4c49dc799e6df07c2b88b115d0668735da15fb3b3862d28d33fa68165", strip_prefix = "mkl-dnn-b01e3a55a07be62172e713bcd2644c5176360212", @@ -184,8 +184,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "eigen_archive", urls = [ - "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", + "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", ], sha256 = "61d8b6fc4279dd1dda986fb1677d15e3d641c07a3ea5abe255790b1f0c0c14e9", strip_prefix = "eigen-eigen-429aa5254200", -- GitLab From 4ec6f2b07c08ddab479541cad0c61f169c1f816f Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 09:59:21 -0700 Subject: [PATCH 265/573] Switching contrib.summaries API to be context-manager-centric PiperOrigin-RevId: 173129793 --- tensorflow/contrib/summary/summary_ops.py | 33 ++++++- .../contrib/summary/summary_ops_test.py | 89 ++++++++++--------- tensorflow/python/eager/context.py | 6 +- 3 files changed, 82 insertions(+), 46 deletions(-) diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index ba3619bfc9..30a9398ee5 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import ops from tensorflow.python.layers import utils from tensorflow.python.ops import summary_op_util from tensorflow.python.training import training_util +from tensorflow.python.util import tf_contextlib # Name for a collection which is expected to have at most a single boolean # Tensor. If this tensor is True the summary ops will record summaries. @@ -46,22 +47,50 @@ def should_record_summaries(): # TODO(apassos) consider how to handle local step here. +@tf_contextlib.contextmanager def record_summaries_every_n_global_steps(n): """Sets the should_record_summaries Tensor to true if global_step % n == 0.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) + old = collection_ref[:] collection_ref[:] = [training_util.get_global_step() % n == 0] + yield + collection_ref[:] = old +@tf_contextlib.contextmanager def always_record_summaries(): """Sets the should_record_summaries Tensor to always true.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) + old = collection_ref[:] collection_ref[:] = [True] + yield + collection_ref[:] = old +@tf_contextlib.contextmanager def never_record_summaries(): """Sets the should_record_summaries Tensor to always false.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) + old = collection_ref[:] collection_ref[:] = [False] + yield + collection_ref[:] = old + + +class SummaryWriter(object): + + def __init__(self, resource): + self._resource = resource + + def set_as_default(self): + context.context().summary_writer_resource = self._resource + + @tf_contextlib.contextmanager + def as_default(self): + old = context.context().summary_writer_resource + context.context().summary_writer_resource = self._resource + yield + context.context().summary_writer_resource = old def create_summary_file_writer(logdir, @@ -77,9 +106,11 @@ def create_summary_file_writer(logdir, if filename_suffix is None: filename_suffix = constant_op.constant("") resource = gen_summary_ops.summary_writer(shared_name=name) + # TODO(apassos) ensure the initialization op runs when in graph mode; consider + # calling session.run here. gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, flush_secs, filename_suffix) - context.context().summary_writer_resource = resource + return SummaryWriter(resource) def _nothing(): diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index 2cd4fce5b3..405a92a726 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -41,60 +41,65 @@ class TargetTest(test_util.TensorFlowTestCase): def testShouldRecordSummary(self): self.assertFalse(summary_ops.should_record_summaries()) - summary_ops.always_record_summaries() - self.assertTrue(summary_ops.should_record_summaries()) + with summary_ops.always_record_summaries(): + self.assertTrue(summary_ops.should_record_summaries()) def testSummaryOps(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() - summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t0') - summary_ops.always_record_summaries() - summary_ops.generic('tensor', 1, '') - summary_ops.scalar('scalar', 2.0) - summary_ops.histogram('histogram', [1.0]) - summary_ops.image('image', [[[[1.0]]]]) - summary_ops.audio('audio', [[1.0]], 1.0, 1) - # The working condition of the ops is tested in the C++ test so we just - # test here that we're calling them correctly. - self.assertTrue(gfile.Exists(logdir)) + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name='t0').as_default(), summary_ops.always_record_summaries(): + summary_ops.generic('tensor', 1, '') + summary_ops.scalar('scalar', 2.0) + summary_ops.histogram('histogram', [1.0]) + summary_ops.image('image', [[[[1.0]]]]) + summary_ops.audio('audio', [[1.0]], 1.0, 1) + # The working condition of the ops is tested in the C++ test so we just + # test here that we're calling them correctly. + self.assertTrue(gfile.Exists(logdir)) def testDefunSummarys(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() - summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t1') - summary_ops.always_record_summaries() - - @function.defun - def write(): - summary_ops.scalar('scalar', 2.0) - - write() - - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list(tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].simple_value, 2.0) + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name='t1').as_default(), summary_ops.always_record_summaries(): + + @function.defun + def write(): + summary_ops.scalar('scalar', 2.0) + + write() + + self.assertTrue(gfile.Exists(logdir)) + files = gfile.ListDirectory(logdir) + self.assertEqual(len(files), 1) + records = list( + tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) + self.assertEqual(len(records), 2) + event = event_pb2.Event() + event.ParseFromString(records[1]) + self.assertEqual(event.summary.value[0].simple_value, 2.0) def testSummaryName(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() - summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t2') - summary_ops.always_record_summaries() - - summary_ops.scalar('scalar', 2.0) - - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list(tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].tag, 'scalar') + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name='t2').as_default(), summary_ops.always_record_summaries(): + + summary_ops.scalar('scalar', 2.0) + + self.assertTrue(gfile.Exists(logdir)) + files = gfile.ListDirectory(logdir) + self.assertEqual(len(files), 1) + records = list( + tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) + self.assertEqual(len(records), 2) + event = event_pb2.Event() + event.ParseFromString(records[1]) + self.assertEqual(event.summary.value[0].tag, 'scalar') if __name__ == '__main__': diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index c5eedb7c9c..92f4e15c05 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -58,6 +58,7 @@ class _EagerContext(threading.local): self.mode = _default_mode self.scope_name = "" self.recording_summaries = False + self.summary_writer_resource = None self.scalar_cache = {} @@ -86,7 +87,6 @@ class Context(object): self._eager_context = _EagerContext() self._context_handle = None self._context_devices = None - self._summary_writer_resource = None self._post_execution_callbacks = [] self._config = config self._seed = None @@ -213,12 +213,12 @@ class Context(object): @property def summary_writer_resource(self): """Returns summary writer resource.""" - return self._summary_writer_resource + return self._eager_context.summary_writer_resource @summary_writer_resource.setter def summary_writer_resource(self, resource): """Sets summary writer resource.""" - self._summary_writer_resource = resource + self._eager_context.summary_writer_resource = resource @property def device_name(self): -- GitLab From 3ed049b673e1a2b58e197fd8429ed81a015cd351 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 09:59:25 -0700 Subject: [PATCH 266/573] Allows calling keras layers in eager mode. PiperOrigin-RevId: 173129805 --- tensorflow/python/keras/_impl/keras/engine/topology.py | 3 +++ tensorflow/python/keras/_impl/keras/layers/core_test.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/tensorflow/python/keras/_impl/keras/engine/topology.py b/tensorflow/python/keras/_impl/keras/engine/topology.py index c0be023b36..f9be782f85 100644 --- a/tensorflow/python/keras/_impl/keras/engine/topology.py +++ b/tensorflow/python/keras/_impl/keras/engine/topology.py @@ -26,6 +26,7 @@ import os import numpy as np from six.moves import zip # pylint: disable=redefined-builtin +from tensorflow.python.eager import context from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras.utils import conv_utils @@ -250,6 +251,8 @@ class Layer(tf_base_layers.Layer): """ # Actually call the layer (optionally building it). output = super(Layer, self).__call__(inputs, **kwargs) + if context.in_eager_mode(): + return output # Update learning phase info. output_tensors = _to_list(output) diff --git a/tensorflow/python/keras/_impl/keras/layers/core_test.py b/tensorflow/python/keras/_impl/keras/layers/core_test.py index 5b15895c41..9cdebd375c 100644 --- a/tensorflow/python/keras/_impl/keras/layers/core_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/core_test.py @@ -20,8 +20,11 @@ from __future__ import print_function import numpy as np +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils +from tensorflow.python.ops import init_ops from tensorflow.python.platform import test @@ -198,6 +201,12 @@ class CoreLayersTest(test.TestCase): self.assertEqual(layer.kernel.constraint, k_constraint) self.assertEqual(layer.bias.constraint, b_constraint) + def test_eager_dense(self): + with context.eager_mode(): + l = keras.layers.Dense(units=3, + kernel_initializer=init_ops.zeros_initializer()) + self.assertAllEqual(l(constant_op.constant([[1.0]])), [[0., 0., 0.]]) + def test_activity_regularization(self): with self.test_session(): layer = keras.layers.ActivityRegularization(l1=0.1) -- GitLab From 0e56ffb7b7cddaf3f0521747d2fade90c56b586f Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Mon, 23 Oct 2017 10:26:34 -0700 Subject: [PATCH 267/573] Fix breakages in OSS builds See example breakages logs at: http://ci.tensorflow.org/job/tensorflow-cl-cpu-python3-pip/10847/console http://ci.tensorflow.org/job/tensorflow-cl-gpu/11008/console 1. CL/172477381 added the no_oss tag to tests with oss_serial tags, which broke the logic of OSS_SERIAL tests in pip.sh and run_pip_test.sh. This CL fixes that. 2. The nccl_kernels BUILD target in contrib/nccl/BUILD was missing some dependencies. This CL adds the missing ones. Fixes: #13918 PiperOrigin-RevId: 173133914 --- tensorflow/contrib/nccl/BUILD | 2 ++ tensorflow/tools/ci_build/builds/run_pip_tests.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD index 5e7263ff62..3aa3215a5f 100644 --- a/tensorflow/contrib/nccl/BUILD +++ b/tensorflow/contrib/nccl/BUILD @@ -74,8 +74,10 @@ tf_kernel_library( "kernels/nccl_rewrite.cc", ], deps = [ + "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:gpu_headers_lib", + "//tensorflow/core:lib", "//tensorflow/core:proto_text", "@nccl_archive//:nccl", ], diff --git a/tensorflow/tools/ci_build/builds/run_pip_tests.sh b/tensorflow/tools/ci_build/builds/run_pip_tests.sh index 43d5c5ff3b..29680e6882 100755 --- a/tensorflow/tools/ci_build/builds/run_pip_tests.sh +++ b/tensorflow/tools/ci_build/builds/run_pip_tests.sh @@ -78,6 +78,7 @@ ln -s $(pwd)/tensorflow ${PIP_TEST_ROOT}/tensorflow # tests with no_pip_gpu tag. PIP_TEST_FILTER_TAG="-no_pip,-no_oss" if [[ ${IS_OSS_SERIAL} == "1" ]]; then + PIP_TEST_FILTER_TAG="$(echo "${PIP_TEST_FILTER_TAG}" | sed s/-no_oss//)" PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},oss_serial" else PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},-oss_serial" -- GitLab From 57f3e529d935e6b08a6c0a3a418ad367d9314fde Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 10:44:59 -0700 Subject: [PATCH 268/573] Internal change PiperOrigin-RevId: 173136642 --- tensorflow/compiler/xla/service/cpu/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index c71eca0d39..136cbe7cb7 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -525,6 +525,7 @@ cc_library( tf_cc_test( name = "cpu_runtime_test", srcs = ["cpu_runtime_test.cc"], + tags = ["optonly"], deps = [ ":cpu_runtime", ":runtime_matmul", -- GitLab From 1038927c096ecc81ca48665871d1be390444b121 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Mon, 23 Oct 2017 11:07:10 -0700 Subject: [PATCH 269/573] Add SerializeIterator op that serializes an IteratorResource into a variant tensor. Add DeserializeIterator op that builds IteratorResource from a variant tensor. Move BundleReaderWrapper and BundleWriterWrapper from dataset.h to iterator_ops.cc. Add generic key-value store interfaces IteratorStateReader and IteratorStateWriter for reading/writing state of iterators. Get rid of IteratorBundleReader and IteratorBundleWriter. PiperOrigin-RevId: 173140858 --- .../contrib/data/python/kernel_tests/BUILD | 4 + .../python/kernel_tests/iterator_ops_test.py | 29 +- .../kernel_tests/range_dataset_op_test.py | 67 ++-- .../kernel_tests/reader_dataset_ops_test.py | 25 +- tensorflow/core/BUILD | 1 + tensorflow/core/framework/iterator.proto | 17 + tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/dataset.h | 189 ++++------ tensorflow/core/kernels/iterator_ops.cc | 355 +++++++++++++----- tensorflow/core/kernels/parse_tensor_op.cc | 1 + tensorflow/core/kernels/range_dataset_op.cc | 11 +- tensorflow/core/kernels/reader_dataset_ops.cc | 17 +- tensorflow/core/kernels/repeat_dataset_op.cc | 13 +- .../core/ops/compat/ops_history.v1.pbtxt | 24 -- tensorflow/core/ops/dataset_ops.cc | 42 ++- tensorflow/python/kernel_tests/BUILD | 5 + .../python/kernel_tests/iterator_ops_test.py | 29 +- .../kernel_tests/range_dataset_op_test.py | 67 ++-- .../kernel_tests/reader_dataset_ops_test.py | 26 +- 19 files changed, 537 insertions(+), 386 deletions(-) create mode 100644 tensorflow/core/framework/iterator.proto diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index c34c9dad9b..b3175e3e56 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -185,6 +185,7 @@ py_test( "//tensorflow/python:function", "//tensorflow/python:functional_ops", "//tensorflow/python:gradients", + "//tensorflow/python:io_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:script_ops", @@ -252,6 +253,8 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", + "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:tensor_shape", "//tensorflow/python:variables", @@ -274,6 +277,7 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", "//tensorflow/python:lib", "//tensorflow/python:parsing_ops", "//tensorflow/python:tensor_shape", diff --git a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py index 20f6d6ba34..bda9a2a4a3 100644 --- a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py @@ -35,6 +35,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import io_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import script_ops @@ -538,9 +539,23 @@ class IteratorTest(test.TestCase): def testIncorrectIteratorRestore(self): - def _iterator_checkpoint_prefix(): + def _path(): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + _path(), parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_range_dataset_graph(): start = 1 stop = 10 @@ -548,22 +563,18 @@ class IteratorTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = _iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op def _build_reader_dataset_graph(): filenames = ["test"] # Does not exist but we don't care in this test. - path = _iterator_checkpoint_prefix() iterator = readers.FixedLengthRecordDataset( filenames, 1, 0, 0).make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op # Saving iterator for RangeDataset graph. diff --git a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py index c8a0072809..c944eb4a49 100644 --- a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py @@ -29,6 +29,8 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops +from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -193,6 +195,21 @@ class RangeDatasetTest(test.TestCase): def _iterator_checkpoint_prefix(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_prefix(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def testSaveRestore(self): def _build_graph(start, stop): @@ -200,10 +217,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -246,14 +261,13 @@ class RangeDatasetTest(test.TestCase): def testRestoreWithoutBuildingDatasetGraph(self): - def _build_graph(start, stop, num_epochs, path): + def _build_graph(start, stop, num_epochs): dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -262,10 +276,8 @@ class RangeDatasetTest(test.TestCase): num_epochs = 5 break_point = 5 break_epoch = 3 - path = self._iterator_checkpoint_prefix() with ops.Graph().as_default() as g: - init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs, - path) + init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs) with self.test_session(graph=g) as sess: sess.run(variables.global_variables_initializer()) sess.run(init_op) @@ -282,8 +294,7 @@ class RangeDatasetTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + restore_op = self._restore_op(iterator._iterator_resource) get_next = iterator.get_next() with self.test_session(graph=g) as sess: sess.run(restore_op) @@ -302,10 +313,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -343,10 +352,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -379,10 +386,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -424,10 +429,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -471,10 +474,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py index c9f88f3dfc..2682e8bdfa 100644 --- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py @@ -33,6 +33,7 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.lib.io import python_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -276,18 +277,31 @@ class FixedLengthRecordReaderTest(test.TestCase): def _iterator_checkpoint_path(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_path(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_iterator_graph(self, num_epochs): filenames = self._createFiles() - path = self._iterator_checkpoint_path() dataset = (readers.FixedLengthRecordDataset( filenames, self._record_bytes, self._header_bytes, self._footer_bytes) .repeat(num_epochs)) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op def _restore_iterator(self): @@ -295,8 +309,7 @@ class FixedLengthRecordReaderTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) get_next = iterator.get_next() - restore_op = gen_dataset_ops.restore_iterator( - iterator._iterator_resource, self._iterator_checkpoint_path()) + restore_op = self._restore_op(iterator._iterator_resource) return restore_op, get_next def testSaveRestore(self): diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 6ad93a73f4..c4f880da9d 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -163,6 +163,7 @@ CORE_PROTO_SRCS = [ "framework/function.proto", "framework/graph.proto", "framework/graph_transfer_info.proto", + "framework/iterator.proto", "framework/kernel_def.proto", "framework/log_memory.proto", "framework/node_def.proto", diff --git a/tensorflow/core/framework/iterator.proto b/tensorflow/core/framework/iterator.proto new file mode 100644 index 0000000000..7e5f5ea2e0 --- /dev/null +++ b/tensorflow/core/framework/iterator.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package tensorflow; +option cc_enable_arenas = true; +option java_outer_classname = "IteratorProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.util"; + +// Protocol buffer representing the metadata for an iterator's state stored +// as a Variant tensor. +message IteratorStateMetadata { + // A user-specified version string. + string version = 1; + + // Keys for tensors in the VariantTensorDataProto. + repeated string keys = 2; +} diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index d931f12f6d..f5bfa60199 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -6061,6 +6061,7 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", ], ) diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h index f9ffc4e065..a906113466 100644 --- a/tensorflow/core/kernels/dataset.h +++ b/tensorflow/core/kernels/dataset.h @@ -17,12 +17,14 @@ limitations under the License. #include +#include "tensorflow/core/common_runtime/graph_runner.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/resource_mgr.h" #include "tensorflow/core/framework/variant_encode_decode.h" #include "tensorflow/core/framework/variant_tensor_data.h" #include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/graph_def_builder.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -39,54 +41,25 @@ namespace tensorflow { class ResourceMgr; -class BundleReaderWrapper { +// Interface for reading values from a key-value store. +// Used for restoring iterator state. +class IteratorStateReader { public: - BundleReaderWrapper(BundleReader* bundle_reader) - : bundle_reader_(bundle_reader) {} + virtual Status ReadScalar(StringPiece key, int64* val) = 0; + virtual Status ReadScalar(StringPiece key, string* val) = 0; + virtual bool Contains(StringPiece key) = 0; - // Reads a scalar value. - template - Status ReadScalar(StringPiece key, T* val) { - Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); - TF_RETURN_IF_ERROR(Lookup(key, &val_t)); - *val = val_t.scalar()(); - return Status::OK(); - } - - bool Contains(StringPiece key) { return bundle_reader_->Contains(key); } - - private: - Status Lookup(StringPiece key, Tensor* val) { - return bundle_reader_->Lookup(key, val); - } - - BundleReader* bundle_reader_; + virtual ~IteratorStateReader() {} }; -class BundleWriterWrapper { +// Interface for writing values to a key-value store. +// Used for saving iterator state. +class IteratorStateWriter { public: - // Note: We intentionally do not provide a constructor that builds a - // BundleWriter from the checkpoint path because we want the caller to be - // in-charge of calling BundleWriter::Finish(). If we expose the Finish() - // method here it may be called pre-maturely by users of this object. - explicit BundleWriterWrapper(BundleWriter* bundle_writer) - : bundle_writer_(bundle_writer) {} - - // Writes a scalar value. - template - Status WriteScalar(StringPiece key, const T val) { - Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); - val_t.scalar()() = val; - TF_RETURN_IF_ERROR(Add(key, val_t)); - return Status::OK(); - } + virtual Status WriteScalar(StringPiece key, const int64& val) = 0; + virtual Status WriteScalar(StringPiece key, const string& val) = 0; - private: - Status Add(StringPiece key, const Tensor& val) { - return bundle_writer_->Add(key, val); - } - - BundleWriter* bundle_writer_; + virtual ~IteratorStateWriter() {} }; // Wrapper around GraphDefBuilder. Used to serialize Dataset graph. @@ -249,10 +222,6 @@ class IteratorContext { // range of outputs is typically represented by an `DatasetBase`, // defined below. class IteratorBase { - protected: - class IteratorBundleReader; - class IteratorBundleWriter; - public: virtual ~IteratorBase() {} @@ -284,87 +253,53 @@ class IteratorBase { virtual const std::vector& output_shapes() const = 0; // Saves the state of this iterator. - virtual Status Save(OpKernelContext* ctx, const string& path) { - BundleWriter bundle_writer(ctx->env(), path); - TF_RETURN_IF_ERROR(bundle_writer.status()); - IteratorBundleWriter writer(&bundle_writer); - TF_RETURN_IF_ERROR(Save(ctx, &writer)); - return bundle_writer.Finish(); + virtual Status Save(IteratorStateWriter* writer) { + if (is_exhausted_) { + LOG(INFO) << "Iterator exhausted."; + return writer->WriteScalar(kIteratorExhausted, kIteratorExhausted); + } else { + return SaveInternal(writer); + } } - virtual Status Restore(OpKernelContext* ctx, const string& path) { - if (!(ctx->env()->FileExists(MetaFilename(path)).ok())) { - return errors::NotFound( - "Failed to restore Iterator state. No file found at ", - MetaFilename(path)); + // Restores the state of this iterator. + virtual Status Restore(OpKernelContext* ctx, IteratorStateReader* reader) { + if (reader->Contains(kIteratorExhausted)) { + LOG(INFO) << "Iterator exhausted. Nothing to restore."; + is_exhausted_ = true; + return Status::OK(); + } else { + return RestoreInternal(ctx, reader); } - BundleReader bundle_reader(ctx->env(), path); - TF_RETURN_IF_ERROR(bundle_reader.status()); - IteratorBundleReader reader(&bundle_reader); - return Restore(ctx, &reader); } static const char kIteratorExhausted[]; protected: // This is needed so that sub-classes of IteratorBase can call - // `RestoreInternal` on their parent iterators, e.g., in + // `SaveInternal` on their parent iterators, e.g., in // `RepeatDataasetOp::Dataset`. - class IteratorBundleReader : public BundleReaderWrapper { - public: - IteratorBundleReader(BundleReader* bundle_reader) - : BundleReaderWrapper(bundle_reader) {} - - // Restores the state of a parent iterator recursively. - Status RestoreParent(OpKernelContext* ctx, - const std::unique_ptr& parent) { - return parent->RestoreInternal(ctx, this); - } - }; + Status SaveParent(IteratorStateWriter* writer, + const std::unique_ptr& parent) { + return parent->SaveInternal(writer); + } // This is needed so that sub-classes of IteratorBase can call - // `SaveInternal` on their parent iterators, e.g., in + // `RestoreInternal` on their parent iterators, e.g., in // `RepeatDataasetOp::Dataset`. - class IteratorBundleWriter : public BundleWriterWrapper { - public: - IteratorBundleWriter(BundleWriter* bundle_writer) - : BundleWriterWrapper(bundle_writer) {} - // Saves the state of a parent iterator recursively. - Status SaveParent(OpKernelContext* ctx, - const std::unique_ptr& parent) { - return parent->SaveInternal(ctx, this); - } - }; - - virtual Status Save(OpKernelContext* ctx, IteratorBundleWriter* writer) { - if (is_exhausted_) { - LOG(INFO) << "Iterator exhausted."; - return writer->WriteScalar(kIteratorExhausted, - kIteratorExhausted); - } else { - return SaveInternal(ctx, writer); - } + Status RestoreParent(OpKernelContext* ctx, IteratorStateReader* reader, + const std::unique_ptr& parent) { + return parent->RestoreInternal(ctx, reader); } - // Saves the state of this iterator. - virtual Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) { + // Saves the state of this iterator recursively. + virtual Status SaveInternal(IteratorStateWriter* writer) { return errors::Unimplemented("SaveInternal"); } - virtual Status Restore(OpKernelContext* ctx, IteratorBundleReader* reader) { - if (reader->Contains(kIteratorExhausted)) { - LOG(INFO) << "Iterator exhausted. Nothing to restore."; - is_exhausted_ = true; - return Status::OK(); - } else { - return RestoreInternal(ctx, reader); - } - } - - // Restores the state of this iterator. + // Restores the state of this iterator recursively. virtual Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) { + IteratorStateReader* reader) { return errors::Unimplemented("RestoreInternal"); } @@ -404,7 +339,7 @@ class DatasetBase : public core::RefCounted { virtual string DebugString() = 0; // Serializes the dataset and writes it to the `writer`. - virtual Status Save(BundleWriterWrapper* writer) const { + virtual Status Save(IteratorStateWriter* writer) const { return errors::Unimplemented("DatasetBase::Save"); } @@ -435,20 +370,14 @@ class GraphDatasetBase : public DatasetBase { const string op_name() const { return op_name_; } - Status Save(BundleWriterWrapper* writer) const override { - GraphDefBuilder b; - DatasetGraphDefBuilder db(&b); - Node* node = nullptr; - TF_RETURN_IF_ERROR(AsGraphDefInternal(&db, &node)); - string output_name = node->name(); - GraphDef graph_def; - TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); + Status Save(IteratorStateWriter* writer) const override { string serialized_graph_def; - graph_def.SerializeToString(&serialized_graph_def); + string output_node; + TF_RETURN_IF_ERROR(Serialize(&serialized_graph_def, &output_node)); TF_RETURN_IF_ERROR( - writer->WriteScalar(kDatasetGraphKey, serialized_graph_def)); + writer->WriteScalar(kDatasetGraphKey, serialized_graph_def)); TF_RETURN_IF_ERROR( - writer->WriteScalar(kDatasetGraphOutputNodeKey, output_name)); + writer->WriteScalar(kDatasetGraphOutputNodeKey, output_node)); return Status::OK(); } @@ -460,6 +389,18 @@ class GraphDatasetBase : public DatasetBase { static const char kDatasetGraphOutputNodeKey[]; private: + Status Serialize(string* serialized_graph_def, string* output_node) const { + GraphDefBuilder b; + DatasetGraphDefBuilder db(&b); + Node* node = nullptr; + TF_RETURN_IF_ERROR(AsGraphDefInternal(&db, &node)); + *output_node = node->name(); + GraphDef graph_def; + TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); + graph_def.SerializeToString(serialized_graph_def); + return Status::OK(); + } + const string op_name_; }; @@ -505,18 +446,18 @@ class DatasetIterator : public IteratorBase { return GetNextInternal(ctx, out_tensors, end_of_sequence); } - protected: - Status Save(OpKernelContext* ctx, IteratorBundleWriter* writer) final { + Status Save(IteratorStateWriter* writer) final { TF_RETURN_IF_ERROR(dataset()->Save(writer)); - return IteratorBase::Save(ctx, writer); + return IteratorBase::Save(writer); } + protected: // Internal implementation of GetNext that is wrapped in tracing logic. virtual Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, bool* end_of_sequence) = 0; - string full_name(const string& name) { + string full_name(const string& name) const { return strings::StrCat(prefix(), ":", name); } diff --git a/tensorflow/core/kernels/iterator_ops.cc b/tensorflow/core/kernels/iterator_ops.cc index df13edc83a..b7c1fff2a9 100644 --- a/tensorflow/core/kernels/iterator_ops.cc +++ b/tensorflow/core/kernels/iterator_ops.cc @@ -16,9 +16,11 @@ limitations under the License. #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/common_runtime/graph_runner.h" +#include "tensorflow/core/framework/iterator.pb.h" #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/resource_op_kernel.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/threadpool.h" @@ -35,6 +37,8 @@ namespace { // See documentation in ../ops/dataset_ops.cc for a high-level // description of the following ops. +const char kIteratorVariantTypeName[] = "tensorflow::Iterator"; + Status VerifyTypesMatch(const DataTypeVector& expected, const DataTypeVector& received) { if (expected.size() != received.size()) { @@ -93,10 +97,10 @@ class IteratorResource : public ResourceBase { } } - Status Save(OpKernelContext* ctx, const string& path) { + Status Save(IteratorStateWriter* writer) { std::shared_ptr captured_iterator(iterator_); if (captured_iterator) { - return captured_iterator->Save(ctx, path); + return captured_iterator->Save(writer); } else { return errors::FailedPrecondition( "Save() failed because the iterator has not been initialized. " @@ -105,49 +109,34 @@ class IteratorResource : public ResourceBase { } } - Status Restore(OpKernelContext* ctx, const string& path) { - if (!(ctx->env()->FileExists(MetaFilename(path)).ok())) { - return errors::NotFound( - "Failed to restore Iterator state. No file found at ", - MetaFilename(path)); - } - - BundleReader bundle_reader(ctx->env(), path); - TF_RETURN_IF_ERROR(bundle_reader.status()); - BundleReaderWrapper reader(&bundle_reader); - if (reader.Contains(GraphDatasetBase::kDatasetGraphKey)) { - string serialized_graph_def; - TF_RETURN_IF_ERROR(reader.ReadScalar(GraphDatasetBase::kDatasetGraphKey, - &serialized_graph_def)); - GraphDef graph_def; - graph_def.ParseFromString(serialized_graph_def); - // TODO(srbs): Is there a way of getting the op registry of the original - // graph. - Graph graph(OpRegistry::Global()); - TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr)); - string output_node; - TF_RETURN_IF_ERROR(reader.ReadScalar( - GraphDatasetBase::kDatasetGraphOutputNodeKey, &output_node)); - std::vector outputs; - GraphRunner graph_runner(ctx->env()); - TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {}, - {output_node}, &outputs)); - DatasetBase* dataset; - TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset)); - TF_RETURN_IF_ERROR(set_iterator(dataset->MakeIterator("Iterator"))); - } else if (reader.Contains(IteratorBase::kIteratorExhausted)) { - TF_RETURN_IF_ERROR(set_iterator(std::unique_ptr( - new ExhaustedIterator(output_dtypes_, output_shapes_)))); + Status Restore(OpKernelContext* ctx, IteratorStateReader* reader) { + string serialized_graph_def; + TF_RETURN_IF_ERROR(reader->ReadScalar(GraphDatasetBase::kDatasetGraphKey, + &serialized_graph_def)); + GraphDef graph_def; + if (!graph_def.ParseFromString(serialized_graph_def)) { + return errors::Internal("Error parsing dataset GraphDef."); } + string output_node; + TF_RETURN_IF_ERROR(reader->ReadScalar( + GraphDatasetBase::kDatasetGraphOutputNodeKey, &output_node)); + DatasetBase* dataset = nullptr; + Graph graph(OpRegistry::Global()); + TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr)); + std::vector outputs; + GraphRunner graph_runner(ctx->env()); + TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {}, + {output_node}, &outputs)); + TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset)); + + TF_RETURN_IF_ERROR(set_iterator(dataset->MakeIterator("Iterator"))); std::shared_ptr captured_iterator(iterator_); if (captured_iterator) { - // TODO(srbs): Figure a way to pass bundle_reader here. - return captured_iterator->Restore(ctx, path); + return captured_iterator->Restore(ctx, reader); } else { return errors::FailedPrecondition( - "Failed to restore iterator from ", path, - ". Make sure the checkpoint ", + "Failed to restore iterator. Make sure the checkpoint ", "is not corrupt. If the checkpoint does not contain the GraphDef, ", "you will need to initialize your iterator before restoring."); } @@ -174,43 +163,194 @@ class IteratorResource : public ResourceBase { } private: - // A no-op iterator which always sets end_of_sequence = true. An instance of - // this is returned when attempting to restore an exhausted iterator. This is - // needed because the Dataset GraphDef may not have been saved for exhausted - // iterators so the actual Iterator can not be built. - class ExhaustedIterator : public IteratorBase { - public: - ExhaustedIterator(const DataTypeVector& output_dtypes, - const std::vector& output_shapes) - : output_dtypes_(output_dtypes), output_shapes_(output_shapes) {} - Status GetNext(IteratorContext* ctx, std::vector* out_tensors, - bool* end_of_sequence) final { - *end_of_sequence = true; - return Status::OK(); - } + std::shared_ptr iterator_; + const DataTypeVector output_dtypes_; + const std::vector output_shapes_; +}; + +// Helper class for reading data from a VariantTensorData object. +class VariantTensorDataReader : public IteratorStateReader { + public: + explicit VariantTensorDataReader(const VariantTensorData* data) + : data_(data) { + PreProcess(); + } + + // Returns OK iff the initialization was successful, i.e., + // pre-processing did not have errors. + Status status() const { return status_; } + + Status ReadScalar(StringPiece key, int64* val) override { + return ReadScalarInternal(key, val); + } + + Status ReadScalar(StringPiece key, string* val) override { + return ReadScalarInternal(key, val); + } - const DataTypeVector& output_dtypes() const override { - return output_dtypes_; + bool Contains(StringPiece key) override { + return map_.find(key.ToString()) != map_.end(); + } + + private: + void PreProcess() { + string metadata; + data_->get_metadata(&metadata); + IteratorStateMetadata proto; + if (!proto.ParseFromString(metadata)) { + status_ = errors::Internal("Error parsing IteratorStateMetadata."); + return; + } + size_t num_entries = proto.keys_size(); + CHECK_EQ(num_entries, data_->tensors_size()); + for (size_t i = 0; i < num_entries; i++) { + map_[proto.keys(i)] = i; } + } - const std::vector& output_shapes() const override { - return output_shapes_; + template + Status ReadScalarInternal(StringPiece key, T* val) { + if (map_.find(key.ToString()) == map_.end()) { + return errors::NotFound(key); } + *val = data_->tensors(map_[key.ToString()]).scalar()(); + return Status::OK(); + } - virtual const std::vector& output_shapes() { - return output_shapes_; + std::map map_; + const VariantTensorData* data_; // Not owned. + Status status_; +}; + +// Helper class for writing data to a VariantTensorData object. +class VariantTensorDataWriter : public IteratorStateWriter { + public: + // Does not take ownership of data. + explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {} + + Status WriteScalar(StringPiece key, const int64& val) override { + return WriteScalarInternal(key, val); + } + + Status WriteScalar(StringPiece key, const string& val) override { + return WriteScalarInternal(key, val); + } + + // Writes the metadata to `data_`. + Status Flush() { + string metadata; + if (!metadata_proto_.SerializeToString(&metadata)) { + return errors::Internal("Unable to serialize IteratorStateMetadata."); } + data_->set_metadata(metadata); + return Status::OK(); + } - private: - const DataTypeVector output_dtypes_; - const std::vector output_shapes_; - }; + private: + template + Status WriteScalarInternal(StringPiece key, const T& val) { + // Write key to the metadata proto. This gets written to `data_` + // when `Flush()` is called. We do this lazily to avoid multiple + // serialization calls. + metadata_proto_.add_keys(key.ToString()); + + // Update tensors. + Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); + val_t.scalar()() = val; + *(data_->add_tensors()) = std::move(val_t); + return Status::OK(); + } - std::shared_ptr iterator_; - const DataTypeVector output_dtypes_; - const std::vector output_shapes_; + VariantTensorData* data_; + // TODO(srbs): Set the version string. + IteratorStateMetadata metadata_proto_; +}; + +// Wrapper for encoding/decoding the iterator state stored in a Variant tensor. +// The get() method returns an IteratorStateReader which can be used +// to restore iterator state. +// +// Usage example: +// +// Encoding: +// +// Tensor t(DT_VARIANT, TensorShape({})); +// t->scalar()() = IteratorStateVariant(iterator_resource); +// +// Encode() sets the type_name of the VariantTensorData object to +// IteratorStateVariant::TypeName(). +// +// Decoding: +// +// Variant v = ; +// DecodeUnaryVariant(&v); +// IteratorStateVariant* wrapper = v.get(); +// iterator_resource->Restore(ctx, wrapper->get()) +// +// The type_name of the VariantTensorData object to be decoded must +// match IteratorStateVariant::TypeName(). +class IteratorStateVariant { + public: + IteratorStateVariant() : data_(nullptr) {} + IteratorStateVariant(const IteratorStateVariant& other) : data_(nullptr) { + if (other.data_) { + Decode(*other.data_); + } + } + // Initializes this object with the current state of the iterator so + // that it can be written on the next call to Encode(). + Status InitializeFromIterator(IteratorResource* iterator_resource) { + data_.reset(new VariantTensorData()); + data_->set_type_name(TypeName()); + VariantTensorDataWriter writer(data_.get()); + TF_RETURN_IF_ERROR(iterator_resource->Save(&writer)); + TF_RETURN_IF_ERROR(writer.Flush()); + return Status::OK(); + } + string TypeName() const { return kIteratorVariantTypeName; } + void Encode(VariantTensorData* data) const { *data = *data_; } + bool Decode(const VariantTensorData& data) { + if (data.type_name() != TypeName()) { + return false; + } + std::unique_ptr tensor_data(new VariantTensorData); + *tensor_data = data; + std::unique_ptr reader( + new VariantTensorDataReader(tensor_data.get())); + status_ = reader->status(); + if (!status_.ok()) { + return false; + } + data_ = std::move(tensor_data); + reader_ = std::move(reader); + return true; + } + IteratorStateReader* get() { return reader_.get(); } + Status status() const { return status_; } + string DebugString() const { + if (data_) { + return strings::StrCat("IteratorStateVariant<", + "data: ", data_->DebugString(), + " status: ", status_.ToString(), ">"); + } else { + return strings::StrCat("IteratorStateVariant"); + } + } + + private: + std::unique_ptr reader_; + Status status_; + std::unique_ptr data_; }; +// Register the reader class in the global variant decode_fn registry +// so that a Variant containing a serialized representation of iterator state +// can be decoded using DecodeUnaryVariant. If we don't do this we will need +// to manually decode the returned Variant using MaybeDecodeAndCopy in +// DeserializeIteratorOp which is not recommended. +REGISTER_UNARY_VARIANT_DECODE_FUNCTION(IteratorStateVariant, + kIteratorVariantTypeName); + // TODO(mrry): Can we simply use the template kernel here? class IteratorHandleOp : public ResourceOpKernel { public: @@ -294,37 +434,6 @@ class ToSingleElementOp : public OpKernel { } }; -class SaveIteratorOp : public OpKernel { - public: - explicit SaveIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - - void Compute(OpKernelContext* ctx) override { - IteratorResource* iterator_resource; - OP_REQUIRES_OK( - ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); - OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->input(1).shape()), - errors::InvalidArgument("SaveIteratorOp: path must be scalar")); - const string& path = ctx->input(1).scalar()(); - OP_REQUIRES_OK(ctx, iterator_resource->Save(ctx, path)); - } -}; - -class RestoreIteratorOp : public OpKernel { - public: - explicit RestoreIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - - void Compute(OpKernelContext* ctx) override { - IteratorResource* iterator_resource; - OP_REQUIRES_OK( - ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); - OP_REQUIRES( - ctx, TensorShapeUtils::IsScalar(ctx->input(1).shape()), - errors::InvalidArgument("RestoreIteratorOp: path must be scalar")); - const string& path = ctx->input(1).scalar()(); - OP_REQUIRES_OK(ctx, iterator_resource->Restore(ctx, path)); - } -}; - class OneShotIteratorOp : public AsyncOpKernel { public: explicit OneShotIteratorOp(OpKernelConstruction* ctx) @@ -644,15 +753,55 @@ class IteratorFromStringHandleOp : public OpKernel { std::vector output_shapes_; }; +class SerializeIteratorOp : public OpKernel { + public: + explicit SerializeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + const Tensor& resource_handle_t = ctx->input(0); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(resource_handle_t.shape()), + errors::InvalidArgument("resource_handle must be a scalar")); + + // Validate that the handle corresponds to a real resource, and + // that it is an IteratorResource. + IteratorResource* iterator_resource; + OP_REQUIRES_OK( + ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); + iterator_resource->Unref(); + Tensor* variant_t; + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &variant_t)); + IteratorStateVariant v; + OP_REQUIRES_OK(ctx, v.InitializeFromIterator(iterator_resource)); + variant_t->scalar()() = v; + } +}; + +class DeserializeIteratorOp : public OpKernel { + public: + explicit DeserializeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + // Validate that the handle corresponds to a real resource, and + // that it is an IteratorResource. + IteratorResource* iterator_resource; + OP_REQUIRES_OK( + ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); + + Variant variant = ctx->input(1).scalar()(); + auto* wrapper = variant.get(); + OP_REQUIRES(ctx, wrapper != nullptr, + errors::InvalidArgument( + "DeserializeIteratorOp: Unable to parse variant tensor.")); + OP_REQUIRES_OK(ctx, wrapper->status()); + OP_REQUIRES_OK(ctx, iterator_resource->Restore(ctx, wrapper->get())); + } +}; + REGISTER_KERNEL_BUILDER(Name("Iterator").Device(DEVICE_CPU), IteratorHandleOp); REGISTER_KERNEL_BUILDER(Name("MakeIterator").Device(DEVICE_CPU), MakeIteratorOp); REGISTER_KERNEL_BUILDER(Name("DatasetToSingleElement").Device(DEVICE_CPU), ToSingleElementOp); -REGISTER_KERNEL_BUILDER(Name("SaveIterator").Device(DEVICE_CPU), - SaveIteratorOp); -REGISTER_KERNEL_BUILDER(Name("RestoreIterator").Device(DEVICE_CPU), - RestoreIteratorOp); REGISTER_KERNEL_BUILDER(Name("OneShotIterator").Device(DEVICE_CPU), OneShotIteratorOp); REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_CPU), @@ -661,6 +810,10 @@ REGISTER_KERNEL_BUILDER(Name("IteratorToStringHandle").Device(DEVICE_CPU), IteratorToStringHandleOp); REGISTER_KERNEL_BUILDER(Name("IteratorFromStringHandle").Device(DEVICE_CPU), IteratorFromStringHandleOp); +REGISTER_KERNEL_BUILDER(Name("SerializeIterator").Device(DEVICE_CPU), + SerializeIteratorOp); +REGISTER_KERNEL_BUILDER(Name("DeserializeIterator").Device(DEVICE_CPU), + DeserializeIteratorOp); } // namespace diff --git a/tensorflow/core/kernels/parse_tensor_op.cc b/tensorflow/core/kernels/parse_tensor_op.cc index ab91a6ef67..6b599612ad 100644 --- a/tensorflow/core/kernels/parse_tensor_op.cc +++ b/tensorflow/core/kernels/parse_tensor_op.cc @@ -92,6 +92,7 @@ class SerializeTensorOp : public OpKernel { Name("SerializeTensor").Device(DEVICE_CPU).TypeConstraint("T"), \ SerializeTensorOp); TF_CALL_ALL_TYPES(REGISTER) +TF_CALL_variant(REGISTER) #undef REGISTER } // namespace tensorflow diff --git a/tensorflow/core/kernels/range_dataset_op.cc b/tensorflow/core/kernels/range_dataset_op.cc index a57c21a590..7adfcc4f8d 100644 --- a/tensorflow/core/kernels/range_dataset_op.cc +++ b/tensorflow/core/kernels/range_dataset_op.cc @@ -112,19 +112,16 @@ class RangeDatasetOp : public DatasetOpKernel { } protected: - Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) override { + Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("next"), next_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("next"), next_)); return Status::OK(); } Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) override { + IteratorStateReader* reader) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("next"), &next_)); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("next"), &next_)); return Status::OK(); } diff --git a/tensorflow/core/kernels/reader_dataset_ops.cc b/tensorflow/core/kernels/reader_dataset_ops.cc index b455c28e07..fb88c55f73 100644 --- a/tensorflow/core/kernels/reader_dataset_ops.cc +++ b/tensorflow/core/kernels/reader_dataset_ops.cc @@ -356,31 +356,30 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel { } protected: - Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) override { + Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar( - full_name("current_file_index"), current_file_index_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"), + current_file_index_)); // `input_buffer_` is empty if // 1. GetNext has not been called even once. // 2. All files have been read and iterator has been exhausted. int64 current_pos = input_buffer_ ? input_buffer_->Tell() : -1; TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("current_pos"), current_pos)); + writer->WriteScalar(full_name("current_pos"), current_pos)); return Status::OK(); } Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) override { + IteratorStateReader* reader) override { mutex_lock l(mu_); int64 current_file_index; - TF_RETURN_IF_ERROR(reader->ReadScalar( - full_name("current_file_index"), ¤t_file_index)); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"), + ¤t_file_index)); current_file_index_ = size_t(current_file_index); int64 current_pos; TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("current_pos"), ¤t_pos)); + reader->ReadScalar(full_name("current_pos"), ¤t_pos)); // Seek to current_pos. input_buffer_.reset(); diff --git a/tensorflow/core/kernels/repeat_dataset_op.cc b/tensorflow/core/kernels/repeat_dataset_op.cc index 5d836927d2..9813e99a70 100644 --- a/tensorflow/core/kernels/repeat_dataset_op.cc +++ b/tensorflow/core/kernels/repeat_dataset_op.cc @@ -124,19 +124,18 @@ class RepeatDatasetOp : public UnaryDatasetOpKernel { } protected: - Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) override { + Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); - TF_RETURN_IF_ERROR(writer->SaveParent(ctx, input_impl_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); + TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_)); return Status::OK(); } Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) override { + IteratorStateReader* reader) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); - TF_RETURN_IF_ERROR(reader->RestoreParent(ctx, input_impl_)); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); + TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_)); return Status::OK(); } diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 6772024263..c5ceb14a09 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -28753,18 +28753,6 @@ op { } is_stateful: true } -op { - name: "RestoreIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - is_stateful: true -} op { name: "RestoreSlice" input_arg { @@ -29548,18 +29536,6 @@ op { } is_stateful: true } -op { - name: "SaveIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - is_stateful: true -} op { name: "SaveSlices" input_arg { diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 566049179a..8b77e3f9f0 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -598,24 +598,6 @@ This operation may be executed multiple times. Each execution will reset the iterator in `iterator` to the first element of `dataset`. )doc"); -REGISTER_OP("SaveIterator") - .Input("iterator: resource") - .Input("path: string") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Saves the state of the `iterator` at `path`. - -This state can be restored using "RestoreIterator". -)doc"); - -REGISTER_OP("RestoreIterator") - .Input("iterator: resource") - .Input("path: string") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Restores the state of the `iterator` from the checkpoint saved at `path` using "SaveIterator". -)doc"); - REGISTER_OP("OneShotIterator") .Output("handle: resource") .Attr("dataset_factory: func") @@ -737,4 +719,28 @@ output_shapes: If specified, defines the shape of each tuple component in an element produced by the resulting iterator. )doc"); +REGISTER_OP("SerializeIterator") + .Input("resource_handle: resource") + .Output("serialized: variant") + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"doc( +Converts the given `resource_handle` representing an iterator to a variant tensor. + +resource_handle: A handle to an iterator resource. +serialized: A variant tensor storing the state of the iterator contained in the + resource. +)doc"); + +REGISTER_OP("DeserializeIterator") + .Input("resource_handle: resource") + .Input("serialized: variant") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Converts the given variant tensor to an iterator and stores it in the given resource. + +resource_handle: A handle to an iterator resource. +serialized: A variant tensor storing the state of the iterator contained in the + resource. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 0e36c3498a..b02bae95fd 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2886,7 +2886,9 @@ tf_py_test( "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:io_ops", "//tensorflow/python:framework_ops", + "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:tensor_shape", "//tensorflow/python:variables", @@ -2907,7 +2909,9 @@ tf_py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", "//tensorflow/python:lib", + "//tensorflow/python:parsing_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:util", "//tensorflow/python/data/ops:iterator_ops", @@ -3022,6 +3026,7 @@ tf_py_test( "//tensorflow/python:function", "//tensorflow/python:functional_ops", "//tensorflow/python:gradients", + "//tensorflow/python:io_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:script_ops", diff --git a/tensorflow/python/kernel_tests/iterator_ops_test.py b/tensorflow/python/kernel_tests/iterator_ops_test.py index b5ec9f7db0..2128ef4ae1 100644 --- a/tensorflow/python/kernel_tests/iterator_ops_test.py +++ b/tensorflow/python/kernel_tests/iterator_ops_test.py @@ -35,6 +35,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import io_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import script_ops @@ -538,9 +539,23 @@ class IteratorTest(test.TestCase): def testIncorrectIteratorRestore(self): - def _iterator_checkpoint_prefix(): + def _path(): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + _path(), parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_range_dataset_graph(): start = 1 stop = 10 @@ -548,22 +563,18 @@ class IteratorTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = _iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op def _build_reader_dataset_graph(): filenames = ["test"] # Does not exist but we don't care in this test. - path = _iterator_checkpoint_prefix() iterator = readers.FixedLengthRecordDataset( filenames, 1, 0, 0).make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op # Saving iterator for RangeDataset graph. diff --git a/tensorflow/python/kernel_tests/range_dataset_op_test.py b/tensorflow/python/kernel_tests/range_dataset_op_test.py index 8291967155..0c530522b8 100644 --- a/tensorflow/python/kernel_tests/range_dataset_op_test.py +++ b/tensorflow/python/kernel_tests/range_dataset_op_test.py @@ -27,6 +27,8 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops +from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -169,6 +171,21 @@ class RangeDatasetTest(test.TestCase): def _iterator_checkpoint_prefix(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_prefix(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def testSaveRestore(self): def _build_graph(start, stop): @@ -176,10 +193,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -222,14 +237,13 @@ class RangeDatasetTest(test.TestCase): def testRestoreWithoutBuildingDatasetGraph(self): - def _build_graph(start, stop, num_epochs, path): + def _build_graph(start, stop, num_epochs): dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -238,10 +252,8 @@ class RangeDatasetTest(test.TestCase): num_epochs = 5 break_point = 5 break_epoch = 3 - path = self._iterator_checkpoint_prefix() with ops.Graph().as_default() as g: - init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs, - path) + init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs) with self.test_session(graph=g) as sess: sess.run(variables.global_variables_initializer()) sess.run(init_op) @@ -258,8 +270,7 @@ class RangeDatasetTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + restore_op = self._restore_op(iterator._iterator_resource) get_next = iterator.get_next() with self.test_session(graph=g) as sess: sess.run(restore_op) @@ -278,10 +289,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -319,10 +328,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -355,10 +362,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -400,10 +405,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -447,10 +450,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 diff --git a/tensorflow/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/kernel_tests/reader_dataset_ops_test.py index 38420328ef..c8e7333b4b 100644 --- a/tensorflow/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/python/kernel_tests/reader_dataset_ops_test.py @@ -31,6 +31,8 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.lib.io import python_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops +from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -273,18 +275,31 @@ class FixedLengthRecordReaderTest(test.TestCase): def _iterator_checkpoint_path(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_path(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_iterator_graph(self, num_epochs): filenames = self._createFiles() - path = self._iterator_checkpoint_path() dataset = (readers.FixedLengthRecordDataset( filenames, self._record_bytes, self._header_bytes, self._footer_bytes) .repeat(num_epochs)) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op def _restore_iterator(self): @@ -292,8 +307,7 @@ class FixedLengthRecordReaderTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) get_next = iterator.get_next() - restore_op = gen_dataset_ops.restore_iterator( - iterator._iterator_resource, self._iterator_checkpoint_path()) + restore_op = self._restore_op(iterator._iterator_resource) return restore_op, get_next def testSaveRestore(self): -- GitLab From 5e23e0e67ac565d56de7680ccb8d8ccc6a0d2179 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 11:12:26 -0700 Subject: [PATCH 270/573] [XLA] Erase cloned instructions on the fly when merging fusion nodes. This avoids the awkward situation where an RNG which is clearly eligible for fusion becomes ineligible mid-fusion because it suddenly has an extra (dead) user. PiperOrigin-RevId: 173141716 --- tensorflow/compiler/xla/service/hlo_instruction.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index f24953051a..0669a86863 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -716,10 +716,12 @@ void HloInstruction::MergeFusionInstructionIntoMultiOutput( // Fuse the root instruction and generate multiple outputs. FuseInstructionIntoMultiOutput(unfused_root); + TF_CHECK_OK(unfused_root->parent()->RemoveInstruction(unfused_root)); // The rest instructions are of normal fusing. for (int64 i = 1; i < unfused_instructions.size(); i++) { auto instruction = unfused_instructions[i]; FuseInstruction(instruction); + TF_CHECK_OK(instruction->parent()->RemoveInstruction(instruction)); } } -- GitLab From 01b6b063811f60764a9604ffcb8cef611f41afa3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 11:29:58 -0700 Subject: [PATCH 271/573] Cut tracing memory cost PiperOrigin-RevId: 173144626 --- .../core/distributed_runtime/master_session.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index 995422644a..f7fce1d0ec 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -746,18 +746,22 @@ void MasterSession::ReffedClientGraph::ProcessStats(int64 step_id, Status::OK()); } // Assemble all stats for this timeline into a merged StepStats. - StepStats step_stats_proto; if (pss->collect_timeline) { - step_stats_proto = pss->rpc_stats; + StepStats step_stats_proto; + step_stats_proto.Swap(&pss->rpc_stats); for (size_t i = 0; i < partitions_.size(); ++i) { - const StepStats& ss = pss->step_stats[i]; - step_stats_proto.MergeFrom(ss); + step_stats_proto.MergeFrom(pss->step_stats[i]); + pss->step_stats[i].Clear(); } - stats_publisher_->PublishStatsProto(step_stats_proto); + pss->step_stats.clear(); // Copy the stats back, but only for on-demand profiling to avoid slowing // down calls that trigger the automatic profiling. if (options.trace_level() == RunOptions::FULL_TRACE) { resp->mutable_step_stats()->Swap(&step_stats_proto); + } else { + // If FULL_TRACE, it can be fetched from Session API, no need for + // duplicated publishing. + stats_publisher_->PublishStatsProto(step_stats_proto); } } } -- GitLab From 9f8523640848f5891d31049b86d5d310cf0f843d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 11:37:06 -0700 Subject: [PATCH 272/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173145770 --- .../core/ops/compat/ops_history.v1.pbtxt | 24 ++++++++ tensorflow/core/ops/ops.pbtxt | 57 ++++++++++--------- 2 files changed, 54 insertions(+), 27 deletions(-) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index c5ceb14a09..92037c1997 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -9582,6 +9582,18 @@ op { } } } +op { + name: "DeserializeIterator" + input_arg { + name: "resource_handle" + type: DT_RESOURCE + } + input_arg { + name: "serialized" + type: DT_VARIANT + } + is_stateful: true +} op { name: "DeserializeManySparse" input_arg { @@ -31465,6 +31477,18 @@ op { } } } +op { + name: "SerializeIterator" + input_arg { + name: "resource_handle" + type: DT_RESOURCE + } + output_arg { + name: "serialized" + type: DT_VARIANT + } + is_stateful: true +} op { name: "SerializeManySparse" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 623f5457bb..c037c99c19 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -6997,6 +6997,21 @@ op { summary: "Dequantize the \'input\' tensor into a float Tensor." description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6. The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / number_of_steps\nconst double offset_input = static_cast(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```" } +op { + name: "DeserializeIterator" + input_arg { + name: "resource_handle" + description: "A handle to an iterator resource." + type: DT_RESOURCE + } + input_arg { + name: "serialized" + description: "A variant tensor storing the state of the iterator contained in the\nresource." + type: DT_VARIANT + } + summary: "Converts the given variant tensor to an iterator and stores it in the given resource." + is_stateful: true +} op { name: "DeserializeManySparse" input_arg { @@ -23025,19 +23040,6 @@ op { description: "Reads a tensor stored in one or several files. If there are several files (for\ninstance because a tensor was saved as slices), `file_pattern` may contain\nwildcard symbols (`*` and `?`) in the filename portion only, not in the\ndirectory portion.\n\nIf a `file_pattern` matches several files, `preferred_shard` can be used to hint\nin which file the requested tensor is likely to be found. This op will first\nopen the file at index `preferred_shard` in the list of matching files and try\nto restore tensors from that file. Only if some tensors or tensor slices are\nnot found in that first file, then the Op opens all the files. Setting\n`preferred_shard` to match the value passed as the `shard` input\nof a matching `Save` Op may speed up Restore. This attribute only affects\nperformance, not correctness. The default value -1 means files are processed in\norder.\n\nSee also `RestoreSlice`." is_stateful: true } -op { - name: "RestoreIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - summary: "Restores the state of the `iterator` from the checkpoint saved at `path` using \"SaveIterator\"." - is_stateful: true -} op { name: "RestoreSlice" input_arg { @@ -23632,20 +23634,6 @@ op { description: "The size of `tensor_names` must match the number of tensors in `data`. `data[i]`\nis written to `filename` with name `tensor_names[i]`.\n\nSee also `SaveSlices`." is_stateful: true } -op { - name: "SaveIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - summary: "Saves the state of the `iterator` at `path`." - description: "This state can be restored using \"RestoreIterator\"." - is_stateful: true -} op { name: "SaveSlices" input_arg { @@ -24990,6 +24978,21 @@ op { } summary: "Computes gradients for the scaled exponential linear (Selu) operation." } +op { + name: "SerializeIterator" + input_arg { + name: "resource_handle" + description: "A handle to an iterator resource." + type: DT_RESOURCE + } + output_arg { + name: "serialized" + description: "A variant tensor storing the state of the iterator contained in the\nresource." + type: DT_VARIANT + } + summary: "Converts the given `resource_handle` representing an iterator to a variant tensor." + is_stateful: true +} op { name: "SerializeManySparse" input_arg { -- GitLab From d25397281cda45e56693979a20d8e622a0b00b29 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 11:45:44 -0700 Subject: [PATCH 273/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173147086 --- tensorflow/go/op/wrappers.go | 224 ++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 108 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index c117711c81..b3b317013f 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -4932,80 +4932,6 @@ func PriorityQueueV2(scope *Scope, shapes []tf.Shape, optional ...PriorityQueueV return op.Output(0) } -// FIFOQueueV2Attr is an optional argument to FIFOQueueV2. -type FIFOQueueV2Attr func(optionalAttr) - -// FIFOQueueV2Shapes sets the optional shapes attribute to value. -// -// value: The shape of each component in a value. The length of this attr must -// be either 0 or the same as the length of component_types. If the length of -// this attr is 0, the shapes of queue elements are not constrained, and -// only one element may be dequeued at a time. -// If not specified, defaults to <> -// -// REQUIRES: len(value) >= 0 -func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["shapes"] = value - } -} - -// FIFOQueueV2Capacity sets the optional capacity attribute to value. -// -// value: The upper bound on the number of elements in this queue. -// Negative numbers mean no limit. -// If not specified, defaults to -1 -func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["capacity"] = value - } -} - -// FIFOQueueV2Container sets the optional container attribute to value. -// -// value: If non-empty, this queue is placed in the given container. -// Otherwise, a default container is used. -// If not specified, defaults to "" -func FIFOQueueV2Container(value string) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// FIFOQueueV2SharedName sets the optional shared_name attribute to value. -// -// value: If non-empty, this queue will be shared under the given name -// across multiple sessions. -// If not specified, defaults to "" -func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// A queue that produces elements in first-in first-out order. -// -// Arguments: -// component_types: The type of each component in a value. -// -// Returns The handle to the queue. -func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"component_types": component_types} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "FIFOQueueV2", - - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // StridedSliceAttr is an optional argument to StridedSlice. type StridedSliceAttr func(optionalAttr) @@ -5385,6 +5311,101 @@ func DynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged return op.Output(0) } +// FIFOQueueV2Attr is an optional argument to FIFOQueueV2. +type FIFOQueueV2Attr func(optionalAttr) + +// FIFOQueueV2Shapes sets the optional shapes attribute to value. +// +// value: The shape of each component in a value. The length of this attr must +// be either 0 or the same as the length of component_types. If the length of +// this attr is 0, the shapes of queue elements are not constrained, and +// only one element may be dequeued at a time. +// If not specified, defaults to <> +// +// REQUIRES: len(value) >= 0 +func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["shapes"] = value + } +} + +// FIFOQueueV2Capacity sets the optional capacity attribute to value. +// +// value: The upper bound on the number of elements in this queue. +// Negative numbers mean no limit. +// If not specified, defaults to -1 +func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["capacity"] = value + } +} + +// FIFOQueueV2Container sets the optional container attribute to value. +// +// value: If non-empty, this queue is placed in the given container. +// Otherwise, a default container is used. +// If not specified, defaults to "" +func FIFOQueueV2Container(value string) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// FIFOQueueV2SharedName sets the optional shared_name attribute to value. +// +// value: If non-empty, this queue will be shared under the given name +// across multiple sessions. +// If not specified, defaults to "" +func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// A queue that produces elements in first-in first-out order. +// +// Arguments: +// component_types: The type of each component in a value. +// +// Returns The handle to the queue. +func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"component_types": component_types} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "FIFOQueueV2", + + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Converts the given `resource_handle` representing an iterator to a variant tensor. +// +// Arguments: +// resource_handle: A handle to an iterator resource. +// +// Returns A variant tensor storing the state of the iterator contained in the +// resource. +func SerializeIterator(scope *Scope, resource_handle tf.Output) (serialized tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SerializeIterator", + Input: []tf.Input{ + resource_handle, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Return a tensor with the same shape and contents as the input tensor or value. func Identity(scope *Scope, input tf.Output) (output tf.Output) { if scope.Err() != nil { @@ -5575,40 +5596,6 @@ func IteratorGetNext(scope *Scope, iterator tf.Output, output_types []tf.DataTyp return components } -// Restores the state of the `iterator` from the checkpoint saved at `path` using "SaveIterator". -// -// Returns the created operation. -func RestoreIterator(scope *Scope, iterator tf.Output, path tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "RestoreIterator", - Input: []tf.Input{ - iterator, path, - }, - } - return scope.AddOperation(opspec) -} - -// Saves the state of the `iterator` at `path`. -// -// This state can be restored using "RestoreIterator". -// -// Returns the created operation. -func SaveIterator(scope *Scope, iterator tf.Output, path tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SaveIterator", - Input: []tf.Input{ - iterator, path, - }, - } - return scope.AddOperation(opspec) -} - // Makes a new iterator from the given `dataset` and stores it in `iterator`. // // This operation may be executed multiple times. Each execution will reset the @@ -5919,6 +5906,27 @@ func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtyp return op.Output(0), op.Output(1) } +// Converts the given variant tensor to an iterator and stores it in the given resource. +// +// Arguments: +// resource_handle: A handle to an iterator resource. +// serialized: A variant tensor storing the state of the iterator contained in the +// resource. +// +// Returns the created operation. +func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "DeserializeIterator", + Input: []tf.Input{ + resource_handle, serialized, + }, + } + return scope.AddOperation(opspec) +} + // Concatenates tensors along one dimension. // // Arguments: -- GitLab From 14506b6a5b26f3351a6d0b8097f0e3d3f2ddf82c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 12:44:54 -0700 Subject: [PATCH 274/573] [TF:XLA] Replace a Mul in XlaArgMinMaxOp with an And PiperOrigin-RevId: 173155697 --- tensorflow/compiler/tf2xla/kernels/BUILD | 1 + .../compiler/tf2xla/kernels/index_ops.cc | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index f44d61de68..4ee7989824 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -83,6 +83,7 @@ tf_kernel_library( "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/compiler/tf2xla/ops:sendrecv_ops", "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:client_library", diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc index db7d556630..b8769b3ea2 100644 --- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" +#include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -82,16 +83,24 @@ void XlaArgMinMaxOp::Compile(XlaOpKernelContext* ctx) { std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0); std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1); // Compute a mask that has 1s for elements equal to the maximum. - xla::ComputationDataHandle mask = b->ConvertElementType( + xla::ComputationDataHandle partial_mask = b->ConvertElementType( b->Eq(input, input_max, broadcast_dims), xla_index_type); - // Multiply by the vector [0, 1, 2, ...] to convert each 1 into its index. - // TODO(phawkins): add a bitwise And operator to HLO, use a bitwise and - // instead of a multiplication here. + // In order to make identity elements for a bitwise And, we: + // Left shift the 1 to the leftmost bit, yielding 0x10...0 + // Arithmetic right shift the 1 back to the rightmost bit, yielding 0xFF...F + int32 bits_in_type = + xla::ShapeUtil::ByteSizeOfPrimitiveType(xla_index_type) * 8 - 1; + xla::ComputationDataHandle shift_amount = + XlaHelpers::IntegerLiteral(b, index_type, bits_in_type); + xla::ComputationDataHandle full_mask = b->ShiftRightArithmetic( + b->ShiftLeft(partial_mask, shift_amount), shift_amount); + + // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its index. xla::ComputationDataHandle iota; OP_REQUIRES_OK(ctx, XlaHelpers::Iota(b, index_type, axis_size, &iota)); xla::ComputationDataHandle product = - b->Mul(mask, iota, /*broadcast_dimensions=*/{axis}); + b->And(full_mask, iota, /*broadcast_dimensions=*/{axis}); // If there are multiple maximum elements, choose the one with the highest // index. -- GitLab From 1a5e8ead650606aba57ec830c82000a6c38e5695 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 13:08:06 -0700 Subject: [PATCH 275/573] Automated g4 rollback of changelist 172936802 PiperOrigin-RevId: 173158990 --- tensorflow/contrib/batching/BUILD | 22 + .../adaptive_shared_batch_scheduler.h | 463 ++++++++++++++++++ .../adaptive_shared_batch_scheduler_test.cc | 438 +++++++++++++++++ tensorflow/contrib/batching/batch_scheduler.h | 2 +- 4 files changed, 924 insertions(+), 1 deletion(-) create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD index 1555a3427f..ae3f48f1b2 100644 --- a/tensorflow/contrib/batching/BUILD +++ b/tensorflow/contrib/batching/BUILD @@ -69,6 +69,28 @@ tf_cc_test( ], ) +cc_library( + name = "adaptive_shared_batch_scheduler", + hdrs = ["adaptive_shared_batch_scheduler.h"], + deps = [ + ":batch_scheduler", + "//tensorflow/contrib/batching/util:periodic_function_dynamic", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "adaptive_shared_batch_scheduler_test", + srcs = ["adaptive_shared_batch_scheduler_test.cc"], + deps = [ + ":adaptive_shared_batch_scheduler", + "//tensorflow/contrib/batching/test_util:fake_clock_env", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + cc_library( name = "basic_batch_scheduler", hdrs = ["basic_batch_scheduler.h"], diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h new file mode 100644 index 0000000000..a0606427a5 --- /dev/null +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h @@ -0,0 +1,463 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ +#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ + +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/batching/batch_scheduler.h" +#include "tensorflow/contrib/batching/util/periodic_function.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace serving { +namespace internal { +template +class ASBSBatch; + +template +class ASBSQueue; +} // namespace internal + +// Shared batch scheduler designed to minimize latency. The scheduler keeps +// track of a number of queues (one per model or model version) which are +// continuously enqueuing requests. The scheduler groups the requests into +// batches which it periodically sends off for processing (see +// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler +// prioritizes batches by age (i.e. the batch's oldest request) irrespective of +// queue. The scheduler will process the oldest batch at an adjustable rate, +// regardless of batch size. The user can provide feedback to help set this rate +// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc). +// +// The rate (or rather, the corresponding period) is adjusted each time a batch +// is processed, using an exponentially weighted moving average to smooth +// potentially noisy feedback: +// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N +// period *= (1 + K * emwa_feedback) +// +// Some potential use cases: +// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing +// involves serial processing by a device, from a latency perspective it is +// desirable to keep the device evenly loaded, avoiding the need to wait for +// the device to process prior batches. +// feedback = num_pending_on_device() - desired_pending. +// CPU utilization - If the batch processing is cpu dominated, you can reap +// latency gains when underutilized by increasing the processing rate, but +// back the rate off when the load increases to avoid overload. +// feedback = cpu_rate() - desired_cpu_rate. + +template +class AdaptiveSharedBatchScheduler + : public std::enable_shared_from_this< + AdaptiveSharedBatchScheduler> { + public: + struct Options { + // The name to use for the pool of batch threads. + string thread_pool_name = {"batch_threads"}; + // Number of batch processing threads; equivalently the maximum number of + // concurrently running batches. + int64 num_batch_threads = port::NumSchedulableCPUs(); + // The environment to use (typically only overridden by test code). + Env* env = Env::Default(); + // Initial batch scheduling period in microseconds. Will be altered for + // non-zero rate_feedback. + double initial_scheduling_period_micros = 500; + // Minimum batch scheduling period in microseconds. Recommend setting this + // value greater than 0, otherwise it may take a while to recover from a + // sustained time of negative scheduling_period_feedback (which may occur + // under low load). + double min_scheduling_period_micros = 100; + // Maximum batch scheduling period in microseconds. + double max_scheduling_period_micros = 10000; + // Feedback function used to modify the scheduling period each time a batch + // is scheduled. Should return values roughly O(1), with positive values + // resulting in an increased period. + std::function scheduling_period_feedback{[] { return 0.; }}; + // To handle potentially noisy scheduling_period_feedback, the period is + // adjusted using an exponentially weighted moving average over the previous + // feedback_smoothing_batches batches. Must be greater than 0. + int64 feedback_smoothing_batches = 10; + }; + + // Ownership is shared between the caller of Create() and any queues created + // via AddQueue(). + static Status Create( + const Options& options, + std::shared_ptr>* scheduler); + + struct QueueOptions { + // Maximum size of each batch. + int max_batch_size = 1000; + // Maximum number of enqueued (i.e. non-scheduled) batches. + int max_enqueued_batches = 10; + }; + + using BatchProcessor = std::function>)>; + + // Adds queue (and its callback) to be managed by this scheduler. + Status AddQueue(const QueueOptions& options, + BatchProcessor process_batch_callback, + std::unique_ptr>* queue); + + private: + // access to AddBatch, RemoveQueue, GetEnv. + friend class internal::ASBSQueue; + + explicit AdaptiveSharedBatchScheduler(const Options& options); + + // Batch scheduling function which runs every scheduling_period_ microseconds. + void ProcessOneBatch(); + + // Notifies scheduler of non-empty batch which is eligible for processing. + void AddBatch(internal::ASBSBatch*); + + // Removes queue from scheduler. + void RemoveQueue(const internal::ASBSQueue* queue); + + Env* GetEnv() const { return options_.env; } + + const Options options_; + + struct BatchCompare { + bool operator()(const internal::ASBSBatch* a, + const internal::ASBSBatch* b); + }; + + // Collection of batches added by AddBatch, ordered by age. Owned by scheduler + // until they are released for processing. + std::priority_queue*, + std::vector*>, BatchCompare> + batches_ GUARDED_BY(mu_); + + // Unowned queues and callbacks added by AddQueue. + std::unordered_map*, BatchProcessor> + queues_and_callbacks_ GUARDED_BY(mu_); + + mutex mu_; + + // Responsible for running ProcessOneBatch. PeriodicFunction was used in order + // to check for deletion so that the thread can be shut down. + std::unique_ptr scheduling_thread_; + + // Responsible for running the batch processing callbacks. + std::unique_ptr batch_thread_pool_; + + // Time interval in microseconds between successive ProcessOneBatch calls. + double scheduling_period_; + + // Exponentially weighted moving average of + // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch + // call. + double ewma_feedback_ = 0; + + TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler); +}; + +////////////////////////////////////////////////////////// +// Implementation details follow. API users need not read. + +namespace internal { +// Consolidates tasks into batches, passing them off to the +// AdaptiveSharedBatchScheduler for processing. +template +class ASBSQueue : public BatchScheduler { + public: + using QueueOptions = + typename AdaptiveSharedBatchScheduler::QueueOptions; + + ASBSQueue(std::shared_ptr> scheduler, + const QueueOptions& options); + + ~ASBSQueue() override; + + // Adds task to current batch. Fails if the task size is larger than the batch + // size or if the current batch is full and this queue's number of outstanding + // batches is at its maximum. + Status Schedule(std::unique_ptr* task) override; + + // Number of tasks waiting to be scheduled. + size_t NumEnqueuedTasks() const override; + + // Number of size 1 tasks which could currently be scheduled without failing. + size_t SchedulingCapacity() const override; + + // Notifies queue that a batch is about to be scheduled; the queue should not + // place any more tasks in this batch. + void ReleaseBatch(const ASBSBatch* batch); + + private: + std::shared_ptr> scheduler_; + const QueueOptions options_; + // Owned by scheduler_. + ASBSBatch* current_batch_ GUARDED_BY(mu_) = nullptr; + int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0; + int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0; + mutable mutex mu_; + TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue); +}; + +// Batch which remembers when and by whom it was created. +template +class ASBSBatch : public Batch { + public: + ASBSBatch(ASBSQueue* queue, int64 creation_time_micros) + : queue_(queue), creation_time_micros_(creation_time_micros) {} + + ~ASBSBatch() override {} + + ASBSQueue* queue() const { return queue_; } + + int64 creation_time_micros() const { return creation_time_micros_; } + + private: + ASBSQueue* queue_; + const int64 creation_time_micros_; + TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch); +}; +} // namespace internal + +// ---------------- AdaptiveSharedBatchScheduler ---------------- + +template +Status AdaptiveSharedBatchScheduler::Create( + const Options& options, + std::shared_ptr>* scheduler) { + if (options.num_batch_threads < 1) { + return errors::InvalidArgument("num_batch_threads must be positive; was ", + options.num_batch_threads); + } + if (options.min_scheduling_period_micros < 0) { + return errors::InvalidArgument( + "min_scheduling_period_micros must be >= 0; was ", + options.min_scheduling_period_micros); + } + if (options.min_scheduling_period_micros > + options.initial_scheduling_period_micros) { + return errors::InvalidArgument( + "initial_scheduling_period_micros (", + options.initial_scheduling_period_micros, + ") must be >= min_scheduling_period_micros (", + options.min_scheduling_period_micros, ")"); + } + if (options.initial_scheduling_period_micros > + options.max_scheduling_period_micros) { + return errors::InvalidArgument( + "initial_scheduling_period_micros (", + options.initial_scheduling_period_micros, + ") must be <= max_scheduling_period_micros (", + options.max_scheduling_period_micros, ")"); + } + if (options.feedback_smoothing_batches < 1) { + return errors::InvalidArgument( + "feedback_smoothing_batches must be positive; was ", + options.feedback_smoothing_batches); + } + scheduler->reset(new AdaptiveSharedBatchScheduler(options)); + return Status::OK(); +} + +template +AdaptiveSharedBatchScheduler::AdaptiveSharedBatchScheduler( + const Options& options) + : options_(options), + scheduling_period_(options.initial_scheduling_period_micros) { + PeriodicFunction::Options opts; + opts.thread_name_prefix = "scheduling_thread"; + opts.env = GetEnv(); + scheduling_thread_.reset( + new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts)); + batch_thread_pool_.reset(new thread::ThreadPool( + GetEnv(), options.thread_pool_name, options.num_batch_threads)); +} + +template +Status AdaptiveSharedBatchScheduler::AddQueue( + const QueueOptions& options, BatchProcessor process_batch_callback, + std::unique_ptr>* queue) { + if (options.max_batch_size <= 0) { + return errors::InvalidArgument("max_batch_size must be positive; was ", + options.max_batch_size); + } + if (options.max_enqueued_batches <= 0) { + return errors::InvalidArgument( + "max_enqueued_batches must be positive; was ", + options.max_enqueued_batches); + } + internal::ASBSQueue* asbs_queue_raw; + queue->reset(asbs_queue_raw = new internal::ASBSQueue( + this->shared_from_this(), options)); + mutex_lock l(mu_); + queues_and_callbacks_[asbs_queue_raw] = process_batch_callback; + return Status::OK(); +} + +template +void AdaptiveSharedBatchScheduler::AddBatch( + internal::ASBSBatch* batch) { + mutex_lock l(mu_); + batches_.push(batch); +} + +template +void AdaptiveSharedBatchScheduler::RemoveQueue( + const internal::ASBSQueue* queue) { + mutex_lock l(mu_); + queues_and_callbacks_.erase(queue); +} + +template +void AdaptiveSharedBatchScheduler::ProcessOneBatch() { + static const double kFeedbackMultiplier = .001; + internal::ASBSBatch* batch = nullptr; + BatchProcessor callback; + const int64 start_time_micros = GetEnv()->NowMicros(); + { + mutex_lock l(mu_); + if (!batches_.empty()) { + batch = batches_.top(); + batches_.pop(); + callback = queues_and_callbacks_[batch->queue()]; + } + } + if (batch != nullptr) { + double feedback = options_.scheduling_period_feedback(); + const int64 N = options_.feedback_smoothing_batches; + ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N; + scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_); + if (scheduling_period_ < options_.min_scheduling_period_micros) { + scheduling_period_ = options_.min_scheduling_period_micros; + } else if (scheduling_period_ > options_.max_scheduling_period_micros) { + scheduling_period_ = options_.max_scheduling_period_micros; + } + // Queue may destroy itself after ReleaseBatch is called. + batch->queue()->ReleaseBatch(batch); + batch_thread_pool_->Schedule([callback, batch] { + callback(std::unique_ptr>(batch)); + }); + } + const int64 sleep_time = + scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros); + if (sleep_time > 0) { + GetEnv()->SleepForMicroseconds(sleep_time); + } +} + +template +bool AdaptiveSharedBatchScheduler::BatchCompare::operator()( + const internal::ASBSBatch* a, + const internal::ASBSBatch* b) { + return a->creation_time_micros() > b->creation_time_micros(); +} + +// ---------------- ASBSQueue ---------------- + +namespace internal { +template +ASBSQueue::ASBSQueue( + std::shared_ptr> scheduler, + const QueueOptions& options) + : scheduler_(scheduler), options_(options) {} + +template +ASBSQueue::~ASBSQueue() { + // Wait until last batch has been scheduled. + const int kSleepMicros = 1000; + for (;;) { + { + mutex_lock l(mu_); + if (num_enqueued_batches_ == 0) { + break; + } + } + scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros); + } + scheduler_->RemoveQueue(this); +} + +template +Status ASBSQueue::Schedule(std::unique_ptr* task) { + bool added_new_batch = false; + size_t size = (*task)->size(); + if (size > options_.max_batch_size) { + return errors::InvalidArgument("Task size ", size, + " is larger than maximum batch size ", + options_.max_batch_size); + } + { + mutex_lock l(mu_); + // Current batch is full, create another if allowed. + if (current_batch_ && + current_batch_->size() + size > options_.max_batch_size) { + if (num_enqueued_batches_ >= options_.max_enqueued_batches) { + return errors::Unavailable("The batch scheduling queue is full"); + } + current_batch_->Close(); + current_batch_ = nullptr; + } + if (!current_batch_) { + added_new_batch = true; + num_enqueued_batches_++; + current_batch_ = + new ASBSBatch(this, scheduler_->GetEnv()->NowMicros()); + } + current_batch_->AddTask(std::move(*task)); + num_enqueued_tasks_++; + } + if (added_new_batch) scheduler_->AddBatch(current_batch_); + return Status::OK(); +} + +template +void ASBSQueue::ReleaseBatch(const ASBSBatch* batch) { + mutex_lock l(mu_); + num_enqueued_batches_--; + num_enqueued_tasks_ -= batch->num_tasks(); + if (batch == current_batch_) { + current_batch_->Close(); + current_batch_ = nullptr; + } +} + +template +size_t ASBSQueue::NumEnqueuedTasks() const { + mutex_lock l(mu_); + return num_enqueued_tasks_; +} + +template +size_t ASBSQueue::SchedulingCapacity() const { + mutex_lock l(mu_); + const int current_batch_capacity = + current_batch_ ? options_.max_batch_size - current_batch_->size() : 0; + const int spare_batches = + options_.max_enqueued_batches - num_enqueued_batches_; + return spare_batches * options_.max_batch_size + current_batch_capacity; +} +} // namespace internal +} // namespace serving +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc new file mode 100644 index 0000000000..a07cd6d834 --- /dev/null +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc @@ -0,0 +1,438 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h" + +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace serving { +namespace anonymous { + +class FakeTask : public BatchTask { + public: + explicit FakeTask(size_t size) : size_(size) {} + + ~FakeTask() override = default; + + size_t size() const override { return size_; } + + private: + const size_t size_; + + TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); +}; + +// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on +// that task. Returns the resulting status. +Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { + std::unique_ptr task(new FakeTask(task_size)); + Status status = scheduler->Schedule(&task); + // Schedule() should have consumed 'task' iff it returned Status::OK. + CHECK_EQ(status.ok(), task == nullptr); + return status; +} + +// Creates a thread that waits on 'start' and then advances the fake clock in +// 'env' in a loop until 'stop' is notified. Useful for allowing objects that +// use the clock to be destroyed. +std::unique_ptr CreateFakeClockAdvancerThread( + test_util::FakeClockEnv* env, Notification* start, Notification* stop) { + return std::unique_ptr(Env::Default()->StartThread( + {}, "FakeClockAdvancerThread", [env, start, stop] { + start->WaitForNotification(); + while (!stop->HasBeenNotified()) { + env->AdvanceByMicroseconds(10); + Env::Default()->SleepForMicroseconds(10); + } + })); +} + +TEST(AdaptiveSharedBatchSchedulerTest, Basic) { + for (const bool delete_scheduler_early : {false, true}) { + for (const bool delete_queue_1_early : {false, true}) { + int queue_0_tasks = 0; + auto queue_0_callback = + [&queue_0_tasks](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_0_tasks += batch->task(i).size(); + } + }; + int queue_1_tasks = 0; + auto queue_1_callback = + [&queue_1_tasks](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_1_tasks += batch->task(i).size(); + } + }; + { + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create({}, &scheduler)); + + // Create two queues. + std::unique_ptr> queue_0; + TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0)); + std::unique_ptr> queue_1; + TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1)); + + if (delete_scheduler_early) { + // Delete our copy of the scheduler. The queues should keep it alive + // under the covers. + scheduler = nullptr; + } + // Submit tasks to the two queues, and (optionally) remove the queues. + TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); + TF_ASSERT_OK(ScheduleTask(3, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(4, queue_1.get())); + if (delete_queue_1_early) { + queue_1 = nullptr; + } + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + } + EXPECT_EQ(queue_0_tasks, 9); + EXPECT_EQ(queue_1_tasks, 6); + } + } +} + +TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) { + using Scheduler = AdaptiveSharedBatchScheduler; + std::shared_ptr scheduler; + Scheduler::Options options; + options.num_batch_threads = 0; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 50; + options.max_scheduling_period_micros = 100; + options.initial_scheduling_period_micros = 1; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 50; + options.max_scheduling_period_micros = 100; + options.initial_scheduling_period_micros = 1000; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 100; + options.max_scheduling_period_micros = 50; + options.initial_scheduling_period_micros = 75; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.feedback_smoothing_batches = 0; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); +} + +TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue_0; + std::unique_ptr> queue_1; + int queue_0_tasks = 0; + int queue_1_tasks = 0; + auto queue_0_callback = [&queue_0_tasks, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_0_tasks += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + auto queue_1_callback = [&queue_1_tasks, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_1_tasks += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + AdaptiveSharedBatchScheduler::QueueOptions queue_options; + queue_options.max_batch_size = 10; + queue_options.max_enqueued_batches = 0; + // Queue must have max_enqueued_batchs > 1. + EXPECT_FALSE( + scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok()); + queue_options.max_enqueued_batches = 2; + TF_ASSERT_OK( + scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); + queue_options.max_batch_size = 0; + // Queue must have max_batch_size > 0. + EXPECT_FALSE( + scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok()); + queue_options.max_batch_size = 2; + queue_options.max_enqueued_batches = 1; + TF_ASSERT_OK( + scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Task larger than max_batch_size shouldn't schedule. + EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok()); + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + env.AdvanceByMicroseconds(1); + + // Task larger than max_batch_size shouldn't schedule. + EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok()); + TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); + TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); + env.AdvanceByMicroseconds(1); + // Exceeds max_enqueued_batches, shouldn't schedule. + EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok()); + + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + // Exceeds max_enqueued_batches, shouldn't schedule. + EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok()); + TF_ASSERT_OK(ScheduleTask(4, queue_0.get())); + + // Batches should be processed in order from oldest to newest. + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 10); + EXPECT_EQ(queue_1_tasks, 0); + + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 10); + EXPECT_EQ(queue_1_tasks, 2); + + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 19); + EXPECT_EQ(queue_1_tasks, 2); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + double feedback = 0; + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.min_scheduling_period_micros = 200; + options.max_scheduling_period_micros = 2000; + options.env = &env; + options.scheduling_period_feedback = [&feedback] { return feedback; }; + options.feedback_smoothing_batches = 1; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + + TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 6 batches. + for (int i = 0; i < 6; i++) { + TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); + env.AdvanceByMicroseconds(1); + } + feedback = -500; + env.AdvanceByMicroseconds(994); + env.BlockUntilThreadsAsleep(2); // scheduling period = 500 usec. + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(500); + env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. + EXPECT_EQ(scheduled_items, 901); + feedback = 0; + env.AdvanceByMicroseconds(250); + env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. + EXPECT_EQ(scheduled_items, 902); + feedback = 10000; // large feedback should hit max_scheduling_period. + env.AdvanceByMicroseconds(250); + env.BlockUntilThreadsAsleep(2); // scheduling period = 2000 usec. + EXPECT_EQ(scheduled_items, 903); + feedback = -10000; // large feedback should hit min_scheduling_period. + env.AdvanceByMicroseconds(1999); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 903); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); // scheduling period = 200 usec. + EXPECT_EQ(scheduled_items, 904); + env.AdvanceByMicroseconds(200); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 905); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + double feedback = 0; + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + options.scheduling_period_feedback = [&feedback] { return feedback; }; + options.feedback_smoothing_batches = 3; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + + TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 4 batches. + for (int i = 0; i < 4; i++) { + TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); + env.AdvanceByMicroseconds(1); + } + feedback = -300; + env.AdvanceByMicroseconds(996); + env.BlockUntilThreadsAsleep(2); + // ewma_feedback = 100, scheduling_period = 900. + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(899); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + // ewma_feedback = 167, scheduling_period = 750. + EXPECT_EQ(scheduled_items, 901); + env.AdvanceByMicroseconds(749); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 901); + feedback = 1000 / 3.; + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + // emwa_feedback = 0, scheduling_period = 750. + EXPECT_EQ(scheduled_items, 902); + env.AdvanceByMicroseconds(749); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 902); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 903); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + AdaptiveSharedBatchScheduler::QueueOptions queue_options; + queue_options.max_batch_size = 10; + queue_options.max_enqueued_batches = 10; + TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 3 tasks. + EXPECT_EQ(queue->NumEnqueuedTasks(), 0); + EXPECT_EQ(queue->SchedulingCapacity(), 100); + TF_ASSERT_OK(ScheduleTask(5, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 1); + EXPECT_EQ(queue->SchedulingCapacity(), 95); + env.AdvanceByMicroseconds(1); + TF_ASSERT_OK(ScheduleTask(6, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 2); + EXPECT_EQ(queue->SchedulingCapacity(), 84); + env.AdvanceByMicroseconds(1); + TF_ASSERT_OK(ScheduleTask(1, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 3); + EXPECT_EQ(queue->SchedulingCapacity(), 83); + + env.AdvanceByMicroseconds(998); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 5); + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 7); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} +} // namespace anonymous +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h index 7c41ad8818..a5072f439a 100644 --- a/tensorflow/contrib/batching/batch_scheduler.h +++ b/tensorflow/contrib/batching/batch_scheduler.h @@ -78,7 +78,7 @@ template class Batch { public: Batch() = default; - ~Batch(); // Blocks until the batch is closed. + virtual ~Batch(); // Blocks until the batch is closed. // Appends 'task' to the batch. After calling AddTask(), the newly-added task // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1). -- GitLab From 2ceaad624dd749bd21ce1cad58143acb9366f297 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 13:08:26 -0700 Subject: [PATCH 276/573] Rename add_layer -> track_layer in Network. PiperOrigin-RevId: 173159032 --- tensorflow/contrib/eager/python/network.py | 16 ++++++++-------- tensorflow/contrib/eager/python/network_test.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index bebc595df0..8ae5099546 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -40,7 +40,7 @@ class Network(base.Layer): print(d.name) print(d.variables) - Note that name provided to __init__ is only for error messages? - - Detect layers used in __call__ that weren't registered with add_layer. + - Detect layers used in __call__ that weren't registered with track_layer. - Convert inputs to __call__ to tensors. - Prevent variables from being created after the first __call__? (Think about restoring from a checkpoint). @@ -52,10 +52,10 @@ class Network(base.Layer): self._container = uuid.uuid4().hex self._layers = collections.OrderedDict() - def add_layer(self, layer): - """Add a Layer to this Network. + def track_layer(self, layer): + """Track a Layer in this Network. - `Network` requires that all `Layer`s used in `call()` be added so that the + `Network` requires that all `Layer`s used in `call()` be tracked so that the `Network` can export a complete list of variables. Args: @@ -66,14 +66,14 @@ class Network(base.Layer): Raises: RuntimeError: If __init__ has not been called. - TypeError: If layer is the wrong type. - ValueError: If a layer with the same name has already been added. + TypeError: If `layer` is the wrong type. + ValueError: If a `Layer` with the same name has already been added. """ if not hasattr(self, "_layers"): raise RuntimeError("Need to call Network.__init__ before adding layers") if not isinstance(layer, base.Layer): raise TypeError( - "Network.add_layer() passed type %s, not a tf.layers.Layer" % + "Network.track_layer() passed type %s, not a tf.layers.Layer" % (type(layer),)) if layer.name in self._layers: if self._layers[layer.name] is layer: @@ -189,7 +189,7 @@ class Sequential(Network): super(Sequential, self).__init__(name=name) if layers: for l in layers: - self.add_layer(l) + self.track_layer(l) def call(self, inputs): """Call each Layer in the order they were added.""" diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py index f0dcae85ee..f43ce3acda 100644 --- a/tensorflow/contrib/eager/python/network_test.py +++ b/tensorflow/contrib/eager/python/network_test.py @@ -27,7 +27,7 @@ class MyNetwork(network.Network): def __init__(self): super(MyNetwork, self).__init__(name="abcd") - self.l1 = self.add_layer(core.Dense(1, use_bias=False)) + self.l1 = self.track_layer(core.Dense(1, use_bias=False)) def call(self, x): return self.l1(x) @@ -94,7 +94,7 @@ class SequentialTest(test.TestCase): # Add a second layer to the network. l2 = core.Dense(1, use_bias=False) - net.add_layer(l2) + net.track_layer(l2) # Set the second layer's weights so it multiplies by 11 net(constant_op.constant([[2.0]])) # Create l2's variables -- GitLab From 3f96f6f956e0e0a6b960e664db4f1e1f2d9b9967 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 13:35:30 -0700 Subject: [PATCH 277/573] Functions in graph mode can call other functions in graph mode. PiperOrigin-RevId: 173163050 --- tensorflow/python/eager/function.py | 3 +++ tensorflow/python/eager/function_test.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index e675ee8988..5afc9d295e 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -347,6 +347,9 @@ class _GraphModeFunction(object): g = ops.get_default_graph() if self._fdef.name not in g._functions: # pylint: disable=protected-access g._add_function(self._fdef) # pylint: disable=protected-access + for f in self._graph._functions.values(): # pylint: disable=protected-access + if f.name not in g._functions: # pylint: disable=protected-access + g._add_function(f) # pylint: disable=protected-access signature = self._fdef.definition.signature args = list(tensor_inputs) + self._extra_inputs op = g.create_op( diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 33bedb59f3..3722f9dfa5 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -86,6 +86,19 @@ class FunctionTest(test.TestCase): op = call() self.assertAllEqual(sess.run(op), 2.0) + def testGraphModeManyFunctions(self): + with context.graph_mode(), self.test_session(): + + @function.defun + def f(x): + return x * x + + @function.defun + def g(x): + return f(x) + 1 + + self.assertAllEqual(g(constant_op.constant(2.0)).eval(), 5.0) + def testTensorConversionWithDefun(self): @function.defun -- GitLab From a0ee701f73cc80a56b41c2452006e166e0b835e6 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Mon, 23 Oct 2017 13:51:45 -0700 Subject: [PATCH 278/573] Remove contrib/xla_tf_graph/ It isn't needed anymore. PiperOrigin-RevId: 173165310 --- tensorflow/BUILD | 1 - tensorflow/compiler/xla/BUILD | 1 - tensorflow/contrib/xla_tf_graph/BUILD | 67 ----- tensorflow/contrib/xla_tf_graph/README.md | 8 - .../contrib/xla_tf_graph/xla_tf_graph_util.cc | 247 ------------------ .../contrib/xla_tf_graph/xla_tf_graph_util.h | 72 ----- .../xla_tf_graph/xla_tf_graph_util_test.cc | 134 ---------- 7 files changed, 530 deletions(-) delete mode 100644 tensorflow/contrib/xla_tf_graph/BUILD delete mode 100644 tensorflow/contrib/xla_tf_graph/README.md delete mode 100644 tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc delete mode 100644 tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h delete mode 100644 tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc diff --git a/tensorflow/BUILD b/tensorflow/BUILD index d4396bacbf..673e433a8a 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -448,7 +448,6 @@ filegroup( "//tensorflow/contrib/training:all_files", "//tensorflow/contrib/util:all_files", "//tensorflow/contrib/verbs:all_files", - "//tensorflow/contrib/xla_tf_graph:all_files", "//tensorflow/core:all_files", "//tensorflow/core/debug:all_files", "//tensorflow/core/distributed_runtime:all_files", diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index e51bbffcd0..0129c51a09 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -7,7 +7,6 @@ package_group( packages = [ "//tensorflow/compiler/...", "//tensorflow/contrib/tpu/...", - "//tensorflow/contrib/xla_tf_graph/...", ], ) diff --git a/tensorflow/contrib/xla_tf_graph/BUILD b/tensorflow/contrib/xla_tf_graph/BUILD deleted file mode 100644 index 4a3a2de9b5..0000000000 --- a/tensorflow/contrib/xla_tf_graph/BUILD +++ /dev/null @@ -1,67 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -package( - default_visibility = ["//visibility:public"], -) - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -load("//tensorflow:tensorflow.bzl", "tf_cc_test") - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), -) - -cc_library( - name = "xla_tf_graph_util", - srcs = [ - "xla_tf_graph_util.cc", - ], - hdrs = [ - "xla_tf_graph_util.h", - ], - deps = [ - "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/xla:status_macros", - "//tensorflow/compiler/xla/client", - "//tensorflow/compiler/xla/client:client_library", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], -) - -tf_cc_test( - name = "xla_tf_graph_util_test", - srcs = ["xla_tf_graph_util_test.cc"], - linkstatic = 1, - tags = ["nomac"], # b/63908145 - deps = [ - ":xla_tf_graph_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/cc:function_ops", - "//tensorflow/cc:scope", - "//tensorflow/compiler/jit:xla_cpu_jit", - "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla/client:client_library", - "//tensorflow/compiler/xla/service:hlo_module_config", - "//tensorflow/core:core_cpu_internal", - "//tensorflow/core:framework_internal", - "//tensorflow/core:ops", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core/kernels:cwise_op", - ], -) diff --git a/tensorflow/contrib/xla_tf_graph/README.md b/tensorflow/contrib/xla_tf_graph/README.md deleted file mode 100644 index a374189e81..0000000000 --- a/tensorflow/contrib/xla_tf_graph/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Xla Tf Graph - -## Description - -This module contains utilities to treat xla representation as tf graph to support mobile SOC experiments and leverage tf tools. - -Maintainers: -- Satoshi Kataoka (satok@google.com, github.com/satok16) diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc deleted file mode 100644 index 302aa6457a..0000000000 --- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc +++ /dev/null @@ -1,247 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h" - -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/client_library.h" -#include "tensorflow/core/platform/protobuf.h" - -namespace tensorflow { -namespace xla_tf_graph { - -namespace { - -constexpr const char* const GRAPH_NAME = "xla_tf_graph"; -constexpr const char* const NODE_NAME_PREFIX = "xla"; - -Status ConvertPrimitiveTypeToDataType(const xla::PrimitiveType p_type, - DataType* d_type) { - switch (p_type) { - case xla::PRED: - *d_type = DT_BOOL; - return Status::OK(); - case xla::S8: - *d_type = DT_INT8; - return Status::OK(); - case xla::S16: - *d_type = DT_INT16; - return Status::OK(); - case xla::S32: - *d_type = DT_INT32; - return Status::OK(); - case xla::S64: - *d_type = DT_INT64; - return Status::OK(); - case xla::U8: - *d_type = DT_UINT8; - return Status::OK(); - case xla::U16: - *d_type = DT_UINT16; - return Status::OK(); - case xla::F16: - *d_type = DT_HALF; - return Status::OK(); - case xla::F32: - *d_type = DT_FLOAT; - return Status::OK(); - case xla::F64: - *d_type = DT_DOUBLE; - return Status::OK(); - default: - return errors::InvalidArgument( - "Unsupported PrimitiveType in ConvertPrimitiveTypeToDataType ", - xla::PrimitiveType_Name(p_type)); - } -} - -Status ConvertXlaShapeToTensorShapeType(const xla::Shape& xla_shape, - std::vector* tensor_shapes, - std::vector* data_types) { - switch (xla_shape.element_type()) { - case xla::TUPLE: { - for (const xla::Shape& element_shape : xla_shape.tuple_shapes()) { - if (element_shape.element_type() == xla::TUPLE) { - return errors::InvalidArgument("Nested tuple is not allowed."); - } - TF_RETURN_IF_ERROR(ConvertXlaShapeToTensorShapeType( - element_shape, tensor_shapes, data_types)); - } - return Status::OK(); - } - case xla::PRED: - case xla::S8: - case xla::S16: - case xla::S32: - case xla::S64: - case xla::U8: - case xla::U16: - case xla::U32: - case xla::U64: - case xla::F16: - case xla::F32: - case xla::F64: { - TensorShape shape; - DataType type; - TF_RETURN_IF_ERROR( - ConvertPrimitiveTypeToDataType(xla_shape.element_type(), &type)); - for (const int64& dim : xla_shape.dimensions()) { - shape.AddDim(dim); - } - tensor_shapes->emplace_back(shape); - data_types->emplace_back(type); - return Status::OK(); - } - default: - return errors::InvalidArgument( - "Unsupported PrimitiveType in ConvertXlaShapeToTensorShapeType ", - xla::PrimitiveType_Name(xla_shape.element_type())); - } -} - -string BuildXlaNodeName(const xla::OperationRequest& operation_request, - const string& xla_op_type, const string& suffix) { - const string name = strings::StrCat( - NODE_NAME_PREFIX, "/", operation_request.output_handle().handle(), "/", - xla_op_type); - if (suffix.empty()) { - return name; - } else { - return strings::StrCat(name, "/", suffix); - } -} - -string BuildXlaNodeName(const xla::OperationRequest& operation_request, - const string& xla_op_type) { - return BuildXlaNodeName(operation_request, xla_op_type, ""); -} - -string BuildXlaNodeOp(const protobuf::Message& msg, const string& suffix) { - return strings::StrCat(msg.GetDescriptor()->name(), "/", suffix); -} - -string BuildXlaNodeOp(const protobuf::Message& msg) { - return BuildXlaNodeOp(msg, ""); -} - -Status ConvertOpRequestToXlaNode(const xla::OperationRequest& operation_request, - XlaNode* xla_node) { - const xla::OpRequest& op_request = operation_request.request(); - switch (op_request.op_case()) { - case xla::OpRequest::kBinaryOpRequest: { - const xla::BinaryOpRequest& op = op_request.binary_op_request(); - xla_node->op_type = - BuildXlaNodeOp(op, xla::BinaryOperation_Name(op.binop())); - xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type); - xla_node->input_ids.emplace_back(std::make_tuple(op.lhs().handle(), 0)); - xla_node->input_ids.emplace_back(std::make_tuple(op.rhs().handle(), 0)); - for (const int64& dim : op.broadcast_dimensions()) { - xla_node->broadcast_dimensions.emplace_back(dim); - } - break; - } - case xla::OpRequest::kParameterRequest: { - const xla::ParameterRequest& op = op_request.parameter_request(); - xla_node->op_type = BuildXlaNodeOp(op, ""); - xla_node->name = - BuildXlaNodeName(operation_request, xla_node->op_type, op.name()); - break; - } - case xla::OpRequest::kVariadicOpRequest: { - const xla::VariadicOpRequest& op = op_request.variadic_op_request(); - xla_node->op_type = - BuildXlaNodeOp(op, xla::VariadicOperation_Name(op.varop())); - xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type); - for (const xla::ComputationDataHandle& handle : op.operands()) { - xla_node->input_ids.emplace_back(std::make_tuple(handle.handle(), 0)); - } - break; - } - case xla::OpRequest::kGetTupleElementRequest: { - const xla::GetTupleElementRequest& op = - op_request.get_tuple_element_request(); - xla_node->op_type = BuildXlaNodeOp(op); - xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type); - xla_node->input_ids.emplace_back( - std::make_tuple(op.operand().handle(), op.index())); - break; - } - default: - // TODO(satok): Implement all possible cases. - LOG(FATAL) << "Op request: " << op_request.op_case() - << " is not supported yet."; - break; - } - - CHECK(!xla_node->name.empty()); - CHECK(!xla_node->op_type.empty()); - - TF_RETURN_IF_ERROR(ConvertXlaShapeToTensorShapeType( - operation_request.output_shape(), &xla_node->output_shapes, - &xla_node->output_data_types)); - return Status::OK(); -} - -void SetupXlaCpuClient(std::unique_ptr* flib_def, - std::unique_ptr* compiler) { - xla::Client* client = xla::ClientLibrary::LocalClientOrDie(); - XlaOpRegistry::RegisterCompilationKernels(); - - FunctionDefLibrary flib; - flib_def->reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib)); - - // Setup compiler options - XlaCompiler::Options options; - DeviceType device_type(DEVICE_CPU_XLA_JIT); - options.device_type = &device_type; - options.flib_def = flib_def->get(); - options.client = client; - compiler->reset(new XlaCompiler(options)); -} - -} // namespace - -xla::StatusOr> -ConvertTfGraphToXlaSessionModule(const std::vector& args, - std::unique_ptr graph) { - CHECK(graph); - - std::unique_ptr flib_def; - std::unique_ptr compiler; - - SetupXlaCpuClient(&flib_def, &compiler); - - // Compile graph and build computation - XlaCompiler::CompilationResult result; - TF_CHECK_OK(compiler->CompileGraph(XlaCompiler::CompileOptions(), GRAPH_NAME, - std::move(graph), args, &result)); - - return result.computation->Snapshot(); -} - -xla::StatusOr> -ConvertXlaSessionModuleToXlaNodes(const xla::SessionModule& session_module) { - std::unordered_map xla_nodes; - for (const auto& operation_request : session_module.entry().requests()) { - XlaNode xla_node; - TF_RETURN_IF_ERROR( - ConvertOpRequestToXlaNode(operation_request.second, &xla_node)); - xla_nodes.emplace(operation_request.first, xla_node); - } - return std::move(xla_nodes); -} - -} // namespace xla_tf_graph -} // namespace tensorflow diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h deleted file mode 100644 index e635290851..0000000000 --- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_ -#define TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_ - -#include - -#include "tensorflow/compiler/tf2xla/xla_compiler.h" -#include "tensorflow/compiler/xla/client/client.h" -#include "tensorflow/compiler/xla/status_macros.h" -#include "tensorflow/core/framework/function.h" -#include "tensorflow/core/graph/graph.h" -#include "tensorflow/core/platform/macros.h" - -namespace tensorflow { -namespace xla_tf_graph { - -// A set of utilities to handle xla computation requests. -// These utilities help developers leverage existing tools to work with -// xla computations, also provide a way to support TensorFlow ops by -// implementing xla computations so that they can do experiments on their -// specialized environments. - -// A structure to represent typed attributes of TensorFlow graph node. -// This structure contains op specific attributes as members so that -// we can treat them explicitly. -struct XlaNode { - // Unique node name - string name; - // Op type of xla computation - string op_type; - // List of pair of unique id and port of input node. - // We store this value instead - // of node name in order not to wait for all XlaNodes to be constructed. - std::vector> input_ids; - // Oputput shapes - std::vector output_shapes; - // Output data types - std::vector output_data_types; - - //--------------------------- - // Op specific attributes - // #xla::OpRequest::kBinaryOpRequest - std::vector broadcast_dimensions; -}; - -// Convert a tf graph to a xla session module -xla::StatusOr> -ConvertTfGraphToXlaSessionModule(const std::vector& args, - std::unique_ptr graph); - -// Convert a xla session module to a map to XlaNode from unique id -xla::StatusOr> -ConvertXlaSessionModuleToXlaNodes(const xla::SessionModule& session_module); - -} // namespace xla_tf_graph -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_ diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc deleted file mode 100644 index 144269303e..0000000000 --- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h" -#include "tensorflow/cc/framework/scope.h" -#include "tensorflow/cc/ops/function_ops.h" -#include "tensorflow/cc/ops/standard_ops.h" -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/client_library.h" -#include "tensorflow/compiler/xla/service/hlo_module_config.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace xla_tf_graph { - -static std::unique_ptr BuildAddGraph() { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0); - auto b = ops::_Arg(scope.WithOpName("B"), DT_INT32, 1); - // See tf2xla/kernels/binary_ops.cc - auto c = ops::Add(scope.WithOpName("C"), a, b); - auto d = ops::_Retval(scope.WithOpName("D"), c, 0); - std::unique_ptr graph(new Graph(OpRegistry::Global())); - TF_CHECK_OK(scope.ToGraph(graph.get())); - return graph; -} - -static std::vector BuildAddGraphArguments() { - // Builds a description of the arguments. - std::vector args(2); - args[0].kind = XlaCompiler::Argument::kParameter; - args[0].type = DT_INT32; - // Difference of dimension will add extra broadcast_dimensions. - // broadcast_dimension generates an additional HloInstruction - // in user_computation.cc - args[0].shape = xla::ShapeUtil::MakeShape(xla::S32, {2, 2}); - args[1].kind = XlaCompiler::Argument::kParameter; - args[1].type = DT_INT32; - args[1].shape = xla::ShapeUtil::MakeShape(xla::S32, {2}); - return args; -} - -// CAVEAT: Debug purpose only. -// This function dumps a protobuf string format of HloModule. -static void DumpHloGraphForDebug(const std::vector& args, - std::unique_ptr graph) { - std::unique_ptr flib_def; - std::unique_ptr flr; - std::unique_ptr compiler; - - xla::Client* client = xla::ClientLibrary::LocalClientOrDie(); - XlaOpRegistry::RegisterCompilationKernels(); - - FunctionDefLibrary flib; - flib_def.reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib)); - - // Compiles the graph. - XlaCompiler::Options options; - DeviceType device_type("XLA_CPU_JIT"); - options.device_type = &device_type; - options.client = client; - options.flib_def = flib_def.get(); - compiler.reset(new XlaCompiler(options)); - - // Compile graph - XlaCompiler::CompilationResult result; - TF_CHECK_OK(compiler->CompileGraph(XlaCompiler::CompileOptions(), "dump", - std::move(graph), args, &result)); - - // Convert to hlo - xla::Computation& computation = *result.computation; - - xla::Service* service( - static_cast(xla::ClientLibrary::GetXlaService( - static_cast(client)->platform()))); - const xla::ComputationTracker& computation_tracker = - service->computation_tracker(); - - auto user_computation_status = - computation_tracker.Resolve(computation.handle()); - TF_CHECK_OK(user_computation_status.status()); - auto user_computation = user_computation_status.ConsumeValueOrDie(); - xla::VersionedComputationHandle versioned_handle = - user_computation->GetVersionedHandle(); - std::unique_ptr hlo_module = - std::move(computation_tracker - .BuildHloModule(versioned_handle, xla::HloModuleConfig()) - .ValueOrDie()); - VLOG(1) << "--- DUMP HLO ---"; - VLOG(1) << hlo_module->ToString(); -} - -TEST(XlaTfGraphUtil, ConvertTfGraphToSessionModule) { - // Builds a description of the arguments. - std::vector args = BuildAddGraphArguments(); - std::unique_ptr graph = BuildAddGraph(); - - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr session_module, - ConvertTfGraphToXlaSessionModule(args, std::move(graph))); - - ASSERT_EQ(4, session_module->entry().requests_size()); - - VLOG(1) << "--- DUMP ---"; - VLOG(1) << session_module->DebugString(); - DumpHloGraphForDebug(args, BuildAddGraph()); -} - -TEST(XlaTfGraphUtil, ConvertXlaSessionModuleToXlaNodes) { - std::vector args = BuildAddGraphArguments(); - std::unique_ptr graph = BuildAddGraph(); - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr session_module, - ConvertTfGraphToXlaSessionModule(args, std::move(graph))); - TF_ASSERT_OK_AND_ASSIGN(auto xla_nodes, - ConvertXlaSessionModuleToXlaNodes(*session_module)); - EXPECT_EQ(session_module->entry().requests_size(), xla_nodes.size()); -} - -} // namespace xla_tf_graph -} // namespace tensorflow -- GitLab From 3f30e6424fa3b8e890f9360d1661e61c2d1625a5 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 14:26:02 -0700 Subject: [PATCH 279/573] Makes gradients_function exception-safe. PiperOrigin-RevId: 173170394 --- tensorflow/python/eager/backprop.py | 57 +++++++++++------------- tensorflow/python/eager/backprop_test.py | 15 +++++++ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 9580e84847..9d86ac77f8 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -335,15 +335,18 @@ def implicit_val_and_grad(f): def grad_fn(*args): """Computes the gradient of the wrapped function.""" tape.push_new_tape() - end_node = f(*args) - variables = tape.top_tape_watched_variables() + try: + end_node = f(*args) + variables = tape.top_tape_watched_variables() + finally: + popped_tape = tape.pop_tape() sources = [x.handle for x in variables] if not sources: raise ValueError("no trainable variables were accessed while the " "function was being computed.") grad = imperative_grad.imperative_grad(_default_vspace, - tape.pop_tape(), + popped_tape, nest.flatten(end_node), sources) return end_node, list(zip(grad, variables)) @@ -561,25 +564,12 @@ def val_and_grad_function(f, params=None): def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" - parameter_positions = _get_arg_spec(f, params, args) dy = kwds.pop("dy", None) - if dy is not None: - dy = ops.convert_to_tensor(dy) - assert not kwds, "The gradient function can't take keyword arguments." - tape.push_new_tape() - sources = [] - args = [ - ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] - for i in range(len(args)) - ] - args = _ensure_unique_tensor_objects(parameter_positions, args) - for i in parameter_positions: - sources.append(args[i]) - tape.watch(args[i]) - result = f(*args) - return result, imperative_grad.imperative_grad( - _default_vspace, tape.pop_tape(), nest.flatten(result), sources, - output_gradients=nest.flatten(dy) if dy is not None else None) + if kwds: + raise ValueError("Functions to be differentiated cannot " + "receive keyword arguments.") + val, vjp = make_vjp(f, params)(*args, **kwds) + return val, vjp(dy=dy) return decorated @@ -619,17 +609,20 @@ def make_vjp(f, params=None): parameter_positions = _get_arg_spec(f, params, args) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() - sources = [] - args = [ - ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] - for i in range(len(args)) - ] - args = _ensure_unique_tensor_objects(parameter_positions, args) - for i in parameter_positions: - sources.append(args[i]) - tape.watch(args[i]) - result = f(*args) - t = tape.pop_tape() + try: + sources = [] + args = [ + ops.convert_to_tensor(args[i]) + if i in parameter_positions else args[i] + for i in range(len(args)) + ] + args = _ensure_unique_tensor_objects(parameter_positions, args) + for i in parameter_positions: + sources.append(args[i]) + tape.watch(args[i]) + result = f(*args) + finally: + t = tape.pop_tape() def vjp(dy=None): return imperative_grad.imperative_grad( _default_vspace, t, nest.flatten(result), sources, diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 9ba5913c65..628f254b18 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -389,6 +389,21 @@ class BackpropTest(test.TestCase): grad = backprop.gradients_function(f) self.assertAllEqual(grad(1.0)[0], 2.0) + def testExceptionSafety(self): + + def f(unused_x): + raise ValueError() + + try: + backprop.gradients_function(f)(1.0) + except ValueError: + pass + + def real_f(x): + return x * x + + self.assertAllEqual(backprop.gradients_function(real_f)(1.0)[0], 2.0) + def testMultiValueConvertToTensor(self): x = resource_variable_ops.ResourceVariable( initial_value=array_ops.constant([1.0]), name='x') -- GitLab From 15022cb1564d3cadaa9b676f3e93c6f16a3a298e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 14:35:53 -0700 Subject: [PATCH 280/573] Remove name_scope from convolutional calls. PiperOrigin-RevId: 173171871 --- .../quantize/python/fold_batch_norms_test.py | 12 +++++----- .../python/quantize_parameterized_test.py | 4 ++-- tensorflow/python/layers/convolutional.py | 22 ++++++++----------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py index 5a66b38b15..2cecf68514 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py @@ -101,9 +101,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): scope + '/weights/read', self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) ]) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) + self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) - folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') + folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') self.assertEqual(folded_conv.type, 'Conv2D') self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', inputs.op.name]) @@ -112,7 +112,7 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') self._AssertInputOpsAre(folded_add, [ - scope + '/convolution_Fold', + scope + '/Conv2D_Fold', self._BathNormBiasName(scope, fused_batch_norm) ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] @@ -166,9 +166,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): scope + '/weights/read', self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) ]) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) + self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) - folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') + folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') self.assertEqual(folded_conv.type, 'Conv2D') self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', inputs.op.name]) self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) @@ -176,7 +176,7 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') self._AssertInputOpsAre(folded_add, [ - scope + '/convolution_Fold', + scope + '/Conv2D_Fold', self._BathNormBiasName(scope, fused_batch_norm) ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py index 31fcd66dfb..3e62f95bd6 100644 --- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py +++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py @@ -101,7 +101,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): scope + '/weights/read' ] self._AssertInputOpsAre(weights_quant, expected_inputs) - output_op_name = scope + '/convolution' + output_op_name = scope + '/Conv2D' self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name]) if with_bypass: @@ -407,7 +407,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): ] self._AssertInputOpsAre(weights_quant, expected_inputs) output_op_name = scope + ('/weights_quant/delayed_quant/Switch_1' - if (delay and use_ema) else '/convolution_Fold') + if (delay and use_ema) else '/Conv2D_Fold') self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name]) if with_bypass: diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 9850cd33b0..c983d3803b 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -153,22 +153,18 @@ class _Conv(base.Layer): self.bias = None self.input_spec = base.InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim}) - with ops.name_scope(None, 'convolution', [self.kernel]) as name: - self._convolution_op = nn_ops.Convolution( - input_shape, - filter_shape=self.kernel.get_shape(), - dilation_rate=self.dilation_rate, - strides=self.strides, - padding=self.padding.upper(), - data_format=utils.convert_data_format(self.data_format, - self.rank + 2), - name=name) + self._convolution_op = nn_ops.Convolution( + input_shape, + filter_shape=self.kernel.get_shape(), + dilation_rate=self.dilation_rate, + strides=self.strides, + padding=self.padding.upper(), + data_format=utils.convert_data_format(self.data_format, + self.rank + 2)) self.built = True def call(self, inputs): - # TODO(agarwal): do we need this name_scope ? - with ops.name_scope(None, 'convolution', [inputs, self.kernel]): - outputs = self._convolution_op(inputs, self.kernel) + outputs = self._convolution_op(inputs, self.kernel) if self.use_bias: if self.data_format == 'channels_first': -- GitLab From c319703d3669c9eec51f17ffc5e7e586b9608074 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Mon, 23 Oct 2017 14:36:55 -0700 Subject: [PATCH 281/573] Java: Update release to 1.4.0-rc1 PiperOrigin-RevId: 173172018 --- tensorflow/java/maven/libtensorflow/pom.xml | 2 +- tensorflow/java/maven/libtensorflow_jni/pom.xml | 2 +- tensorflow/java/maven/pom.xml | 2 +- tensorflow/java/maven/proto/pom.xml | 2 +- tensorflow/java/maven/tensorflow/pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml index 6cc1102930..3714570876 100644 --- a/tensorflow/java/maven/libtensorflow/pom.xml +++ b/tensorflow/java/maven/libtensorflow/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.3.0 + 1.4.0-rc1 ../ libtensorflow diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml index 0b22844898..9f7eb40253 100644 --- a/tensorflow/java/maven/libtensorflow_jni/pom.xml +++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.3.0 + 1.4.0-rc1 ../ libtensorflow_jni diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml index 0a3552d756..fac0a8bc26 100644 --- a/tensorflow/java/maven/pom.xml +++ b/tensorflow/java/maven/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.tensorflow parentpom - 1.3.0 + 1.4.0-rc1 pom https://www.tensorflow.org diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml index b76b28aa15..135ee0f2d2 100644 --- a/tensorflow/java/maven/proto/pom.xml +++ b/tensorflow/java/maven/proto/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.3.0 + 1.4.0-rc1 ../ proto diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml index c2af55f5ce..771482ba64 100644 --- a/tensorflow/java/maven/tensorflow/pom.xml +++ b/tensorflow/java/maven/tensorflow/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.3.0 + 1.4.0-rc1 ../ tensorflow -- GitLab From 4cd64cac16ccd22a2d956d6957ecfda0ff67ee89 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 14:40:39 -0700 Subject: [PATCH 282/573] Make Optimizer.minimize work when eager execution is enabled. PiperOrigin-RevId: 173172604 --- tensorflow/python/BUILD | 1 + tensorflow/python/eager/backprop.py | 2 +- tensorflow/python/eager/function_test.py | 2 +- tensorflow/python/framework/test_util.py | 4 +- tensorflow/python/training/optimizer.py | 49 ++++- tensorflow/python/training/optimizer_test.py | 211 +++++++++++-------- 6 files changed, 174 insertions(+), 95 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index b7aa7bbf6b..4382eeb9a8 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2658,6 +2658,7 @@ py_library( ":util", ":variable_scope", ":variables", + "//tensorflow/python/eager:backprop", "//third_party/py/numpy", "@six_archive//:six", ], diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 9d86ac77f8..bdc4ce3252 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -343,7 +343,7 @@ def implicit_val_and_grad(f): sources = [x.handle for x in variables] if not sources: - raise ValueError("no trainable variables were accessed while the " + raise ValueError("No trainable variables were accessed while the " "function was being computed.") grad = imperative_grad.imperative_grad(_default_vspace, popped_tape, diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 3722f9dfa5..b4b704401a 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -138,7 +138,7 @@ class FunctionTest(test.TestCase): 'v', initializer=constant_op.constant(1.0)) return x * constant_op.constant(2.0) with self.assertRaisesRegexp(ValueError, - 'no trainable variables were accessed'): + 'No trainable variables were accessed'): backprop.implicit_val_and_grad(f)() def testDefunCallBackpropUsingSameObjectForMultipleArguments(self): diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index a01bf02deb..e545f6de8e 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -683,8 +683,10 @@ class TensorFlowTestCase(googletest.TestCase): elif isinstance(tensors, dict): assert not tensors, "Only support empty dict now." return dict() + elif tensors is None: + return None else: - raise ValueError("Unsupported type.") + raise ValueError("Unsupported type %s." % type(tensors)) def evaluate(self, tensors): """Evaluates tensors and returns numpy values. diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index 86ba8e2c8e..82fc4edbcd 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -22,6 +22,7 @@ from __future__ import print_function import abc +from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -335,6 +336,16 @@ class Optimizer(object): Raises: ValueError: If some of the variables are not `Variable` objects. + + @compatibility(eager): + When eager execution is enabled, `loss` should be a Python function that + takes elements of `var_list` as arguments and computes the value to be + minimized. If `var_list` is None, `loss` should take no arguments. + Minimization (and gradient computation) is done with respect to the + elements of `var_list` if not None, else with respect to any trainable + variables created during the execution of the `loss` function. + `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and + `grad_loss` are ignored when eager execution is enabled. """ grads_and_vars = self.compute_gradients( loss, var_list=var_list, gate_gradients=gate_gradients, @@ -385,7 +396,32 @@ class Optimizer(object): Raises: TypeError: If `var_list` contains anything else than `Variable` objects. ValueError: If some arguments are invalid. + + @compatibility(eager): + When eager execution is enabled, `loss` should be a Python function that + takes elements of `var_list` as arguments and computes the value to be + minimized. If `var_list` is None, `loss` should take no arguments. + Gradient computation is done with respect to the elements of `var_list` if + not None, else with respect to any trainable variables created during the + execution of the `loss` function. + `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and + `grad_loss` are ignored when eager execution is enabled. """ + if context.in_eager_mode(): + if grad_loss is not None: + raise ValueError("`grad_loss` argument to Optimizer.compute_gradients " + "not supported when eager execution is enabled.") + if not callable(loss): + raise ValueError("`loss` passed to Optimizer.compute_gradients should " + "be a function when eager execution is enabled.") + # TODO(agarwal): consider passing parameters to the `loss` function. + if var_list is None: + return backprop.implicit_grad(loss)() + else: + var_list = nest.flatten(var_list) + grads = backprop.gradients_function(loss)(*var_list) + grads_and_vars = list(zip(grads, var_list)) + return grads_and_vars if gate_gradients not in [Optimizer.GATE_NONE, Optimizer.GATE_OP, Optimizer.GATE_GRAPH]: raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, " @@ -489,11 +525,14 @@ class Optimizer(object): else: with ops.control_dependencies([self._finish(update_ops, "update")]): with ops.colocate_with(global_step): - apply_updates = state_ops.assign_add(global_step, 1, name=name).op - - train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) - if apply_updates not in train_op: - train_op.append(apply_updates) + apply_updates = state_ops.assign_add(global_step, 1, name=name) + + if context.in_graph_mode(): + if isinstance(apply_updates, ops.Tensor): + apply_updates = apply_updates.op + train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) + if apply_updates not in train_op: + train_op.append(apply_updates) return apply_updates diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py index c7eb9bc412..6bdae39073 100644 --- a/tensorflow/python/training/optimizer_test.py +++ b/tensorflow/python/training/optimizer_test.py @@ -18,12 +18,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -32,26 +35,34 @@ from tensorflow.python.training import gradient_descent class OptimizerTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a_%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b_%d' % i) + def loss(v0, v1): + return 5 * v0 + 3 * v1 + # Note that for eager execution, minimize expects a function instead of a + # Tensor. + cost = loss if context.in_eager_mode() else loss(var0, var1) + global_step = resource_variable_ops.ResourceVariable( + array_ops.zeros([], dtypes.int64), name='global_step_%d' % i) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([-14., -13.], var0.eval()) - self.assertAllClose([-6., -5.], var1.eval()) + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + # Run 1 step of sgd through optimizer + opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) + self.evaluate(opt_op) + # Validate updated params + self.assertAllClose([-14., -13.], self.evaluate(var0)) + self.assertAllClose([-6., -5.], self.evaluate(var1)) def testAggregationMethod(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: @@ -103,86 +114,112 @@ class OptimizerTest(test.TestCase): self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)], var1.eval()) + @test_util.run_in_graph_and_eager_modes() def testNoVariables(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype, trainable=False) - var1 = variables.Variable([3.0, 4.0], dtype=dtype, trainable=False) - cost = 5 * var0 + var1 - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, 'No variables'): - sgd_op.minimize(cost) + # pylint: disable=cell-var-from-loop + def loss(): + var0 = resource_variable_ops.ResourceVariable( + [1.0, 2.0], dtype=dtype, trainable=False, name='a') + var1 = resource_variable_ops.ResourceVariable( + [3.0, 4.0], dtype=dtype, trainable=False, name='b') + return 5 * var0 + var1 + # pylint: enable=cell-var-from-loop + cost = loss if context.in_eager_mode() else loss() + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, 'No.*variables'): + sgd_op.minimize(cost) + @test_util.run_in_graph_and_eager_modes() def testNoGradients(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, 'No gradients'): - # var1 has no gradient - sgd_op.minimize(cost, global_step, [var1]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b%d' % i) + # pylint: disable=cell-var-from-loop + def loss(_): + return 5 * var0 + # pylint: enable=cell-var-from-loop + cost = loss if context.in_eager_mode() else loss(var1) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, 'No gradients'): + # var1 has no gradient + sgd_op.minimize(cost, var_list=[var1]) + @test_util.run_in_graph_and_eager_modes() def testNoGradientsForAnyVariables_Minimize(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = constant_op.constant(5.0) - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, - 'No gradients provided for any variable'): - sgd_op.minimize(cost, global_step, [var0, var1]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a_%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b_%d' % i) + def loss(unused_v1, unused_v2): + return constant_op.constant(5.0) + cost = loss if context.in_eager_mode() else loss(var0, var1) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, + 'No gradients provided for any variable'): + sgd_op.minimize(cost, var_list=[var0, var1]) + @test_util.run_in_graph_and_eager_modes() def testNoGradientsForAnyVariables_ApplyGradients(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, - 'No gradients provided for any variable'): - sgd_op.apply_gradients([(None, var0), (None, var1)]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a_%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b_%d' % i) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, + 'No gradients provided for any variable'): + sgd_op.apply_gradients([(None, var0), (None, var1)]) + @test_util.run_in_graph_and_eager_modes() def testGradientsAsVariables(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session() as sess: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1]) - # Convert gradients to tf.Variables - converted_grads = [ - variables.Variable(array_ops.zeros([2], dtype)) - for i in grads_and_vars - ] - convert_ops = [ - state_ops.assign(converted_grads[i], gv[0]) - for i, gv in enumerate(grads_and_vars) - ] - - converted_grads_and_vars = list(zip(converted_grads, [var0, var1])) - opt_op = sgd_op.apply_gradients(converted_grads_and_vars, global_step) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b%d' % i) + def loss(v0, v1): + return 5 * v0 + 3 * v1 + cost = loss if context.in_eager_mode() else loss(var0, var1) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1]) + # Convert gradients to tf.Variables + converted_grads = [ + resource_variable_ops.ResourceVariable(array_ops.zeros([2], dtype), + name='c_%d_%d' % (i, j)) + for j, gv in enumerate(grads_and_vars) + ] + convert_ops = [ + state_ops.assign(converted_grads[j], gv[0]) + for j, gv in enumerate(grads_and_vars) + ] - variables.global_variables_initializer().run() - # Run convert_ops to achieve the gradietns converting - sess.run(convert_ops) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([-14., -13.], var0.eval()) - self.assertAllClose([-6., -5.], var1.eval()) + self.evaluate(variables.global_variables_initializer()) + # Run convert_ops to achieve the gradietns converting + self.evaluate(convert_ops) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 1 step of sgd through optimizer + converted_grads_and_vars = list(zip(converted_grads, [var0, var1])) + opt_op = sgd_op.apply_gradients(converted_grads_and_vars) + self.evaluate(opt_op) + + # Validate updated params + self.assertAllClose([-14., -13.], self.evaluate(var0)) + self.assertAllClose([-6., -5.], self.evaluate(var1)) def testTrainOp(self): with self.test_session(): -- GitLab From 4f127e9019ff32f5c165550d535e4ad0fa587dd6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 14:42:03 -0700 Subject: [PATCH 283/573] Infer `tf.contrib.distributions.RelaxedOneHotCategorical` `dtype` from arguments. PiperOrigin-RevId: 173172808 --- .../relaxed_onehot_categorical_test.py | 8 ++++++ .../python/ops/relaxed_onehot_categorical.py | 25 +++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py index 8c8363fe3f..faae9da6ad 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py @@ -164,6 +164,14 @@ class RelaxedOneHotCategoricalTest(test.TestCase): self.assertAllEqual([5, 3], dist.sample(5).eval(feed_dict=feed_dict).shape) + def testDTypes(self): + # check that sampling and log_prob work for a range of dtypes + with self.test_session(): + for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): + logits = random_ops.random_uniform(shape=[3, 3], dtype=dtype) + dist = relaxed_onehot_categorical.RelaxedOneHotCategorical( + temperature=0.5, logits=logits) + dist.log_prob(dist.sample()) if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py index 699cf45a73..b6becfa9fc 100644 --- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py +++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py @@ -130,7 +130,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): temperature, logits=None, probs=None, - dtype=dtypes.float32, + dtype=None, validate_args=False, allow_nan_stats=True, name="ExpRelaxedOneHotCategorical"): @@ -150,7 +150,8 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): `N - 1` dimensions index into a batch of independent distributions and the last dimension represents a vector of probabilities for each class. Only one of `logits` or `probs` should be passed in. - dtype: The type of the event samples (default: float32). + dtype: The type of the event samples (default: inferred from + logits/probs). validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -163,14 +164,21 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): """ parameters = locals() with ops.name_scope(name, values=[logits, probs, temperature]): + + self._logits, self._probs = distribution_util.get_logits_and_probs( + name=name, logits=logits, probs=probs, validate_args=validate_args, + multidimensional=True) + + if dtype is None: + dtype = self._logits.dtype + if not validate_args: + temperature = math_ops.cast(temperature, dtype) + with ops.control_dependencies([check_ops.assert_positive(temperature)] if validate_args else []): self._temperature = array_ops.identity(temperature, name="temperature") self._temperature_2d = array_ops.reshape(temperature, [-1, 1], name="temperature_2d") - self._logits, self._probs = distribution_util.get_logits_and_probs( - name=name, logits=logits, probs=probs, validate_args=validate_args, - multidimensional=True) logits_shape_static = self._logits.get_shape().with_rank_at_least(1) if logits_shape_static.ndims is not None: @@ -230,7 +238,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): def _sample_n(self, n, seed=None): sample_shape = array_ops.concat([[n], array_ops.shape(self.logits)], 0) - logits = self.logits * array_ops.ones(sample_shape) + logits = self.logits * array_ops.ones(sample_shape, dtype=self.dtype) logits_2d = array_ops.reshape(logits, [-1, self.event_size]) # Uniform variates must be sampled from the open-interval `(0, 1)` rather # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` @@ -368,7 +376,7 @@ class RelaxedOneHotCategorical( temperature, logits=None, probs=None, - dtype=dtypes.float32, + dtype=None, validate_args=False, allow_nan_stats=True, name="RelaxedOneHotCategorical"): @@ -388,7 +396,8 @@ class RelaxedOneHotCategorical( dimensions index into a batch of independent distributions and the last dimension represents a vector of probabilities for each class. Only one of `logits` or `probs` should be passed in. - dtype: The type of the event samples (default: float32). + dtype: The type of the event samples (default: inferred from + logits/probs). validate_args: Unused in this distribution. allow_nan_stats: Python `bool`, default `True`. If `False`, raise an exception if a statistic (e.g. mean/mode/etc...) is undefined for any -- GitLab From f226eb3717a0df815579178f4393d4e68cbe08fc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 14:42:57 -0700 Subject: [PATCH 284/573] [XLA] Adds a C64 type to XLA, with actual compilation support coming soon. PiperOrigin-RevId: 173172916 --- tensorflow/compiler/tf2xla/type_util.cc | 3 + tensorflow/compiler/xla/literal_util.cc | 101 +++++++++++++++++- tensorflow/compiler/xla/literal_util.h | 26 +++++ tensorflow/compiler/xla/literal_util_test.cc | 85 ++++++++++++++- tensorflow/compiler/xla/primitive_util.cc | 18 ++++ tensorflow/compiler/xla/primitive_util.h | 15 +++ .../compiler/xla/service/hlo_evaluator.cc | 3 + tensorflow/compiler/xla/shape_util.cc | 6 ++ tensorflow/compiler/xla/shape_util.h | 3 + tensorflow/compiler/xla/shape_util_test.cc | 4 + .../xla/tests/client_library_test_base.cc | 6 +- .../compiler/xla/tests/literal_test_util.cc | 52 ++++++++- tensorflow/compiler/xla/types.h | 2 + tensorflow/compiler/xla/xla_data.proto | 4 + 14 files changed, 317 insertions(+), 11 deletions(-) diff --git a/tensorflow/compiler/tf2xla/type_util.cc b/tensorflow/compiler/tf2xla/type_util.cc index c698488776..1efbe0ffb1 100644 --- a/tensorflow/compiler/tf2xla/type_util.cc +++ b/tensorflow/compiler/tf2xla/type_util.cc @@ -58,6 +58,9 @@ Status DataTypeToPrimitiveType(DataType data_type, xla::PrimitiveType* type) { case tensorflow::DT_DOUBLE: *type = xla::F64; return Status::OK(); + case tensorflow::DT_COMPLEX64: + *type = xla::C64; + return Status::OK(); case tensorflow::DT_QUINT8: *type = xla::U8; return Status::OK(); diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc index 79e40c1262..413b85e3ba 100644 --- a/tensorflow/compiler/xla/literal_util.cc +++ b/tensorflow/compiler/xla/literal_util.cc @@ -173,6 +173,8 @@ Status Literal::Copy(const Literal& src_literal, return CopyRange(src_literal, src_base, dest_base, copy_size); case F64: return CopyRange(src_literal, src_base, dest_base, copy_size); + case C64: + return CopyRange(src_literal, src_base, dest_base, copy_size); case PRED: return CopyRange(src_literal, src_base, dest_base, copy_size); default: @@ -522,6 +524,10 @@ string Literal::GetAsString( return tensorflow::strings::StrCat(Get(multi_index)); case F64: return tensorflow::strings::StrCat(Get(multi_index)); + case C64: { + complex64 c = Get(multi_index); + return tensorflow::strings::StrCat("(", c.real(), ", ", c.imag(), ")"); + } case F16: return tensorflow::strings::StrCat(Get(multi_index)); default: @@ -716,6 +722,8 @@ void* Literal::MutableInternalData() { return reinterpret_cast(f32s_.data()); case F64: return reinterpret_cast(f64s_.data()); + case C64: + return reinterpret_cast(c64s_.data()); case F16: return reinterpret_cast(f16s_.data()); default: @@ -754,6 +762,9 @@ void Literal::Reserve(int64 num_elements) { case F64: Resize(num_elements, 0); break; + case C64: + Resize(num_elements, 0); + break; case F16: Resize(num_elements, static_cast(0.0f)); break; @@ -790,6 +801,9 @@ tensorflow::Status Literal::ValidateLiteral() const { case F64: actual = f64s_size(); break; + case C64: + actual = c64s_size(); + break; case F16: actual = f16s().size() / sizeof(half); break; @@ -843,6 +857,26 @@ std::unique_ptr ConvertBetweenNativeTypes(const Literal& src_literal) { return result_literal; } +template +std::unique_ptr ConvertToC64(const Literal& src_literal) { + auto result_literal = MakeUnique(); + Shape* result_shape = result_literal->mutable_shape(); + *result_shape = src_literal.shape(); + result_shape->set_element_type(C64); + result_literal->Reserve(ShapeUtil::ElementsIn(*result_shape)); + using NativeSrcT = + typename primitive_util::PrimitiveTypeToNative::type; + tensorflow::gtl::ArraySlice src_data = + src_literal.GetArraySlice(); + tensorflow::gtl::MutableArraySlice dest_data = + result_literal->GetMutableArraySlice(); + int64 num_elements = ShapeUtil::ElementsIn(src_literal.shape()); + for (int64 i = 0; i < num_elements; ++i) { + dest_data[i] = complex64(static_cast(src_data[i]), 0); + } + return result_literal; +} + template std::unique_ptr ConvertIfTypesMatch(const Literal& src_literal) { CHECK_EQ(primitive_src_type, src_literal.shape().element_type()); @@ -870,6 +904,8 @@ StatusOr> ConvertIfDestTypeMatches( CONVERT_IF_TYPES_MATCH(F32) CONVERT_IF_TYPES_MATCH(F64) #undef CONVERT_IF_TYPES_MATCH + case C64: + return ConvertToC64(src_literal); // Other types are not yet supported. default: return InvalidArgument( @@ -966,6 +1002,8 @@ bool Literal::operator==(const Literal& other) const { return EqualElements(*this, other, 0, &multi_index); case F16: return EqualElements(*this, other, 0, &multi_index); + case C64: + return EqualElements(*this, other, 0, &multi_index); default: LOG(FATAL) << "Unimplemented: Literal::Equal for type " << PrimitiveType_Name(shape().element_type()); @@ -1065,6 +1103,12 @@ tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice() { values->size()); } +template <> +tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice() { + auto values = mutable_c64s(); + return {values->data(), values->size()}; +} + template <> tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice() { // TODO - there is an endianess problem here. fix it, or wait for uint16 @@ -1144,6 +1188,13 @@ tensorflow::gtl::ArraySlice Literal::GetArraySlice() const { f16s().size() / sizeof(half)); } +template <> +tensorflow::gtl::ArraySlice Literal::GetArraySlice() + const { + CHECK_EQ(shape().element_type(), C64); + return c64s(); +} + template static bool AllElementsEqualValue(const Literal& literal, NativeT value) { for (int64 i = 0; i < ShapeUtil::ElementsIn(literal.shape()); ++i) { @@ -1211,6 +1262,15 @@ bool Literal::IsAllFloat(float value) const { } } +bool Literal::IsAllComplex(complex64 value) const { + switch (shape().element_type()) { + case C64: + return AllElementsEqualValue(*this, value); + default: + return false; + } +} + bool Literal::IsZero(tensorflow::gtl::ArraySlice indices) const { switch (shape().element_type()) { case U8: @@ -1229,6 +1289,8 @@ bool Literal::IsZero(tensorflow::gtl::ArraySlice indices) const { return Get(indices) == 0.0f; case F64: return Get(indices) == 0.0; + case C64: + return Get(indices) == complex64(0.0f, 0.0f); case F16: return Get(indices) == static_cast(0.0f); case PRED: @@ -1298,12 +1360,27 @@ void Literal::Resize(int64 num_elements, half value) { mutable_f16s()->resize(num_elements, value); } +template <> +void Literal::Resize(int64 num_elements, complex64 value) { + CHECK_EQ(ShapeUtil::ElementsIn(shape()), num_elements); + mutable_c64s()->resize(num_elements, value); +} + template -static void CopyToRepeatedField(RepeatedFieldT* dest, - const std::vector& src) { +void CopyToRepeatedField(RepeatedFieldT* dest, + const std::vector& src) { *dest = RepeatedFieldT(src.begin(), src.end()); } +template <> +void CopyToRepeatedField, complex64>( + tensorflow::protobuf::RepeatedField* dest, + const std::vector& src) { + *dest = tensorflow::protobuf::RepeatedField( + reinterpret_cast(src.data()), + reinterpret_cast(src.data()) + src.size() * 2); +} + LiteralProto Literal::ToProto() const { LiteralProto proto; proto.Clear(); @@ -1338,6 +1415,9 @@ LiteralProto Literal::ToProto() const { case F64: CopyToRepeatedField(proto.mutable_f64s(), f64s()); break; + case C64: + CopyToRepeatedField(proto.mutable_c64s(), c64s()); + break; case TUPLE: for (const auto& tuple : tuple_literals()) { *proto.add_tuple_literals() = tuple.ToProto(); @@ -1351,11 +1431,21 @@ LiteralProto Literal::ToProto() const { } template -static void CopyFromRepeatedField(std::vector* dest, - const RepeatedFieldT& src) { +void CopyFromRepeatedField(std::vector* dest, + const RepeatedFieldT& src) { *dest = std::vector(src.begin(), src.end()); } +template <> +void CopyFromRepeatedField, + complex64>( + std::vector* dest, + const tensorflow::protobuf::RepeatedField& src) { + *dest = std::vector( + reinterpret_cast(src.data()), + reinterpret_cast(src.data()) + src.size() / 2); +} + void Literal::CopyFromProto(const LiteralProto& literal_proto) { if (!literal_proto.has_shape()) { return; @@ -1394,6 +1484,9 @@ void Literal::CopyFromProto(const LiteralProto& literal_proto) { case F64: CopyFromRepeatedField(mutable_f64s(), literal_proto.f64s()); break; + case C64: + CopyFromRepeatedField(mutable_c64s(), literal_proto.c64s()); + break; case TUPLE: for (const auto& proto : literal_proto.tuple_literals()) { mutable_tuple_literals()->push_back(Literal(proto)); diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h index 4063cb05a9..a1e288829f 100644 --- a/tensorflow/compiler/xla/literal_util.h +++ b/tensorflow/compiler/xla/literal_util.h @@ -159,6 +159,10 @@ class Literal { const std::vector& f64s() const { return f64s_; } std::vector* mutable_f64s() { return &f64s_; } + int c64s_size() const { return c64s().size(); } + const std::vector& c64s() const { return c64s_; } + std::vector* mutable_c64s() { return &c64s_; } + int tuple_literals_size() const { return tuple_literals().size(); } const Literal& tuple_literals(int i) const { return tuple_literals_[i]; } Literal* add_tuple_literals() { @@ -560,6 +564,17 @@ class Literal { // e.g. -0.5. bool IsAllFloat(float value) const; + // Like IsAll(const Literal&, int8), except we check whether the literal is + // equal to a particular complex number. + // + // If the literal is not a complex value, this always returns false. + // + // This casts value to the type of literal, then compares using ==. The usual + // admonishments about floating-point equality checks apply. We expect you to + // use this to check for complex values that can be expressed precisely as + // float pairs e.g. (-0.5, 1.0). + bool IsAllComplex(complex64 value) const; + // Returns whether this literal is zero at the specified index. This literal // must be an array. bool IsZero(tensorflow::gtl::ArraySlice indices) const; @@ -610,6 +625,7 @@ class Literal { std::vector f16s_; std::vector f32s_; std::vector f64s_; + std::vector c64s_; std::vector tuple_literals_; }; @@ -658,6 +674,10 @@ tensorflow::gtl::ArraySlice Literal::GetArraySlice() const; template <> tensorflow::gtl::ArraySlice Literal::GetArraySlice() const; +template <> +tensorflow::gtl::ArraySlice Literal::GetArraySlice() + const; + template <> tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); @@ -694,6 +714,9 @@ tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); template <> tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); +template <> +tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); + template <> void Literal::Resize(int64 num_elements, bool value); @@ -724,6 +747,9 @@ void Literal::Resize(int64 num_elements, double value); template <> void Literal::Resize(int64 num_elements, half value); +template <> +void Literal::Resize(int64 num_elements, complex64 value); + template /* static */ std::unique_ptr Literal::CreateR0(NativeT value) { auto literal = MakeUnique(); diff --git a/tensorflow/compiler/xla/literal_util_test.cc b/tensorflow/compiler/xla/literal_util_test.cc index e7dedd0821..a9af4849e2 100644 --- a/tensorflow/compiler/xla/literal_util_test.cc +++ b/tensorflow/compiler/xla/literal_util_test.cc @@ -107,6 +107,9 @@ TEST_F(LiteralUtilTest, LiteralScalarToString) { auto f16_lit = Literal::CreateR0(static_cast(0.5f)); ASSERT_EQ("0.5", f16_lit->ToString()); + + auto c64_lit = Literal::CreateR0({3.14f, 2.78f}); + ASSERT_EQ("(3.14, 2.78)", c64_lit->ToString()); } TEST_F(LiteralUtilTest, LiteralVectorToString) { @@ -331,6 +334,19 @@ TEST_F(LiteralUtilTest, TupleEquality) { EXPECT_NE(*tuple1, *different_tuple); } +TEST_F(LiteralUtilTest, C64Equality) { + // Test equality with tuples. + auto vector = Literal::CreateR1({{1.0, 2.0}, {3.0, 4.0}}); + + // Tuple with the same elements. One element is shared with the original + // tuple, the other is a clone of the element in the original tuple. + auto vector_clone = Literal::CreateR1({{1.0, 2.0}, {3.0, 4.0}}); + EXPECT_EQ(*vector, *vector_clone); + + auto vector_reversed = Literal::CreateR1({{3.0, 4.0}, {1.0, 2.0}}); + EXPECT_NE(*vector, *vector_reversed); +} + TEST_F(LiteralUtilTest, IsAllTuple) { auto element1 = Literal::CreateR0(0.0); auto element2 = Literal::CreateR2({{0.0, 0.0}, {0.0, 0.0}}); @@ -381,6 +397,9 @@ TEST_F(LiteralUtilTest, IsAll) { EXPECT_FALSE(Literal::CreateR2({{h8}, {h9}})->IsAll(8)); EXPECT_FALSE(Literal::CreateR2({{h9}, {h8}})->IsAll(8)); + complex64 c8_9 = {8, 9}; + EXPECT_FALSE(Literal::CreateR2({{c8_9}, {c8_9}})->IsAll(8)); + auto uint64_max = std::numeric_limits::max(); EXPECT_FALSE(Literal::CreateR2( {{uint64_max, uint64_max}, {uint64_max, uint64_max}}) @@ -411,6 +430,25 @@ TEST_F(LiteralUtilTest, IsAllFloat) { Literal::CreateR2({{0, 0, 0}, {0, .1, 0}})->IsAllFloat(0)); } +TEST_F(LiteralUtilTest, IsAllComplex) { + // IsAllComplex always returns false when the literal is not complex. + EXPECT_FALSE(Literal::CreateR0(false)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + + complex64 c8_9 = {8, 9}; + complex64 c7_9 = {7, 9}; + EXPECT_TRUE(Literal::CreateR2({{c8_9}, {c8_9}}) + ->IsAllComplex({8.0f, 9.0f})); + EXPECT_FALSE(Literal::CreateR2({{c7_9}, {c8_9}}) + ->IsAllComplex({8.0f, 9.0f})); + EXPECT_FALSE(Literal::CreateR2({{c8_9}, {c7_9}}) + ->IsAllComplex({8.0f, 9.0f})); +} + TEST_F(LiteralUtilTest, IsZero) { auto scalar_zero = Literal::CreateR0(0.0f); auto scalar_one = Literal::CreateR0(1.0f); @@ -422,12 +460,17 @@ TEST_F(LiteralUtilTest, IsZero) { EXPECT_TRUE(array->IsZero({0, 2})); EXPECT_TRUE(array->IsZero({1, 1})); EXPECT_FALSE(array->IsZero({1, 2})); + + auto complex_zero = Literal::CreateR0(0.0f); + auto complex_nonzero = Literal::CreateR0(0.5f); + EXPECT_TRUE(complex_zero->IsZero({})); + EXPECT_FALSE(complex_nonzero->IsZero({})); } template class LiteralUtilTestTemplated : public ::testing::Test {}; -using TestedTypes = ::testing::Types; +using TestedTypes = ::testing::Types; TYPED_TEST_CASE(LiteralUtilTestTemplated, TestedTypes); TYPED_TEST(LiteralUtilTestTemplated, Relayout2x2) { @@ -626,13 +669,28 @@ TEST_F(LiteralUtilTest, PopulateR1S64) { EXPECT_EQ(output, *expected); } -TEST_F(LiteralUtilTest, PopulateR2U64) { +TEST_F(LiteralUtilTest, PopulateR1U64) { Literal output; output.PopulateR1({{77, 88}}); auto expected = Literal::CreateR1({{77, 88}}); EXPECT_EQ(output, *expected); } +TEST_F(LiteralUtilTest, PopulateR1C64) { + Literal output; + output.PopulateR1({{77, 88}}); + auto expected = Literal::CreateR1({{77, 88}}); + EXPECT_EQ(output, *expected); +} + +TEST_F(LiteralUtilTest, PopulateR2C64) { + Literal output; + output.PopulateR2({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}}); + auto expected = + Literal::CreateR2({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}}); + EXPECT_EQ(output, *expected); +} + TEST_F(LiteralUtilTest, PopulateWithValueR0F32) { Literal output; output.PopulateWithValue(2.5f, {}); @@ -654,6 +712,14 @@ TEST_F(LiteralUtilTest, PopulateWithValueR2U64) { EXPECT_EQ(output, *expected); } +TEST_F(LiteralUtilTest, PopulateWithValueR2C64) { + Literal output; + output.PopulateWithValue({4, 2}, {2, 2}); + auto expected = + Literal::CreateR2({{{4, 2}, {4, 2}}, {{4, 2}, {4, 2}}}); + EXPECT_EQ(output, *expected); +} + TEST_F(LiteralUtilTest, PopulateWithValueR0F16) { Literal output; half h(0.25f); @@ -919,6 +985,11 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) { {{0.0, 19.0, 0.0, 21.0}, {22.0, 0.0, 24.0, 0.0}}, {{26.0, 0.0, 28.0, 0.0}, {0.0, 31.0, 0.0, 33.0}}, }}, layout_r4_dim0major_); + auto c64 = Literal::CreateR4WithLayout({{ + {{10.0f, 0.0f, 12.0f, 0.0f}, {0.0f, 15.0f, 0.0f, 17.0f}}, + {{0.0f, 19.0f, 0.0f, 21.0f}, {22.0f, 0.0f, 24.0f, 0.0f}}, + {{26.0f, 0.0f, 28.0f, 0.0f}, {0.0f, 31.0f, 0.0f, 33.0f}}, + }}, layout_r4_dim0major_); // clang-format on std::unique_ptr conv; @@ -961,12 +1032,22 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) { conv = u32->Convert(F16).ConsumeValueOrDie(); EXPECT_EQ(*conv, *f16); + conv = s32->Convert(C64).ConsumeValueOrDie(); + EXPECT_EQ(*conv, *c64); + + conv = f16->Convert(C64).ConsumeValueOrDie(); + EXPECT_EQ(*conv, *c64); + EXPECT_EQ(s32->Convert(TUPLE).status().code(), tensorflow::error::INVALID_ARGUMENT); EXPECT_EQ(s32->Convert(S16).status().code(), tensorflow::error::INVALID_ARGUMENT); EXPECT_EQ(s32->Convert(U16).status().code(), tensorflow::error::INVALID_ARGUMENT); + EXPECT_EQ(c64->Convert(F32).status().code(), + tensorflow::error::INVALID_ARGUMENT); + EXPECT_EQ(c64->Convert(S32).status().code(), + tensorflow::error::INVALID_ARGUMENT); } TEST_F(LiteralUtilTest, CopyFromProto_Bool) { diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc index e4e37177a2..2113b5e06f 100644 --- a/tensorflow/compiler/xla/primitive_util.cc +++ b/tensorflow/compiler/xla/primitive_util.cc @@ -83,10 +83,17 @@ PrimitiveType NativeToPrimitiveType() { return F16; } +template <> +PrimitiveType NativeToPrimitiveType() { + return C64; +} + bool IsFloatingPointType(PrimitiveType type) { return type == F16 || type == F32 || type == F64; } +bool IsComplexType(PrimitiveType type) { return type == C64; } + bool IsSignedIntegralType(PrimitiveType type) { return type == S8 || type == S16 || type == S32 || type == S64; } @@ -121,6 +128,7 @@ int BitWidth(PrimitiveType type) { case U64: case S64: case F64: + case C64: return 64; case TUPLE: @@ -134,5 +142,15 @@ int BitWidth(PrimitiveType type) { } } +PrimitiveType ComplexComponentType(PrimitiveType complex_type) { + switch (complex_type) { + case C64: + return F32; + default: + LOG(FATAL) << "Primitive type is not complex: " + << PrimitiveType_Name(complex_type); + } +} + } // namespace primitive_util } // namespace xla diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h index 162a11c7d2..a49c8b86fc 100644 --- a/tensorflow/compiler/xla/primitive_util.h +++ b/tensorflow/compiler/xla/primitive_util.h @@ -78,8 +78,14 @@ PrimitiveType NativeToPrimitiveType(); template <> PrimitiveType NativeToPrimitiveType(); +// Complex +template <> +PrimitiveType NativeToPrimitiveType(); + bool IsFloatingPointType(PrimitiveType type); +bool IsComplexType(PrimitiveType type); + bool IsSignedIntegralType(PrimitiveType type); bool IsUnsignedIntegralType(PrimitiveType type); @@ -89,6 +95,10 @@ bool IsIntegralType(PrimitiveType type); // Returns the number of bits in the representation for a given type. int BitWidth(PrimitiveType type); +// Returns the real, imag component type underlying the given complex type. +// LOG(FATAL)'s if complex_type is not complex. +PrimitiveType ComplexComponentType(PrimitiveType complex_type); + // Returns the native type (eg, float) corresponding to the given template // parameter XLA primitive type (eg, F32). template @@ -157,6 +167,11 @@ struct PrimitiveTypeToNative { using type = half; }; +// Complex +template <> +struct PrimitiveTypeToNative { + using type = complex64; +}; } // namespace primitive_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index e8f88427da..fa6a8f3d53 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -1265,6 +1265,9 @@ HloEvaluator::HloEvaluator() { }); typed_visitors_[F32] = MakeUnique>(this); typed_visitors_[F64] = MakeUnique>(this); + typed_visitors_[C64] = MakeUnique([](HloInstruction*) { + return Unimplemented("unhandled primitive type: C64."); + }); typed_visitors_[TUPLE] = MakeUnique([](HloInstruction*) { return Unimplemented("unhandled primitive type: TUPLE."); }); diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index af583bed62..fa4f71414d 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -281,6 +281,10 @@ StatusOr MakeShapeWithLayoutInternal( } } +/* static */ bool ShapeUtil::ElementIsComplex(const Shape& shape) { + return primitive_util::IsComplexType(shape.element_type()); +} + /* static */ bool ShapeUtil::ElementIsFloating(const Shape& shape) { return primitive_util::IsFloatingPointType(shape.element_type()); } @@ -592,6 +596,8 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { return sizeof(float); case F64: return sizeof(double); + case C64: + return sizeof(complex64); default: LOG(FATAL) << "Unhandled primitive type " << primitive_type; } diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index c5800acaf1..8f8d4a73c9 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -291,6 +291,9 @@ class ShapeUtil { // Returns whether the element type of the shape is floating point. static bool ElementIsFloating(const Shape& shape); + // Returns whether the element type of the shape is complex. + static bool ElementIsComplex(const Shape& shape); + // Returns whether the element type has the given bit width. static bool ElementHasBitWidth(const Shape& shape, int bits); diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc index 79945b9c77..0ba542ad1b 100644 --- a/tensorflow/compiler/xla/shape_util_test.cc +++ b/tensorflow/compiler/xla/shape_util_test.cc @@ -218,6 +218,10 @@ TEST(ShapeUtilTest, ByteSizeOfWithoutPadding) { EXPECT_EQ(8, ShapeUtil::ByteSizeOfPrimitiveType(F64)); EXPECT_EQ(8, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(F64, {}))); EXPECT_EQ(1600, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(F64, {10, 20}))); + + EXPECT_EQ(8, ShapeUtil::ByteSizeOfPrimitiveType(C64)); + EXPECT_EQ(8, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(C64, {}))); + EXPECT_EQ(1600, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(C64, {10, 20}))); } TEST(ShapeUtilTest, ByteSizeOfWithPadding) { diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc index a60d3e50bd..065bce7e31 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.cc +++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc @@ -254,7 +254,8 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus( tensorflow::gtl::ArraySlice arguments, const Shape* shape_with_layout) { TF_ASSIGN_OR_RETURN(auto computation, builder->Build()); - if (ShapeUtil::ElementIsFloating(expected.shape())) { + if (ShapeUtil::ElementIsFloating(expected.shape()) || + ShapeUtil::ElementIsComplex(expected.shape())) { LOG(WARNING) << "performing exact comparison of floating point numbers"; } else { TF_RET_CHECK(ShapeUtil::ElementIsIntegral(expected.shape()) || @@ -282,7 +283,8 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus( ComputationBuilder* builder, const Literal& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error, const Shape* shape_with_layout) { - TF_RET_CHECK(ShapeUtil::ElementIsFloating(expected.shape())); + TF_RET_CHECK(ShapeUtil::ElementIsFloating(expected.shape()) || + ShapeUtil::ElementIsComplex(expected.shape())); TF_ASSIGN_OR_RETURN(auto computation, builder->Build()); auto expect_near = [&](const Literal& actual, const string& error_message) { LiteralTestUtil::ExpectNear(expected, actual, error, error_message); diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc index 2876a79dd8..95a52ecd2f 100644 --- a/tensorflow/compiler/xla/tests/literal_test_util.cc +++ b/tensorflow/compiler/xla/tests/literal_test_util.cc @@ -156,6 +156,15 @@ template <> ::testing::AssertionResult CompareEqual(double lhs, double rhs) { return CompareFloatsBitwiseEqual(lhs, rhs); } +template <> +::testing::AssertionResult CompareEqual(complex64 lhs, + complex64 rhs) { + auto res = CompareEqual(lhs.real(), rhs.real()); + if (!res) { + return res; + } + return CompareEqual(lhs.imag(), rhs.imag()); +} // A recursive function which iterates through every index of expected and // actual literal and compares their values elementwise. Returns true if all @@ -235,6 +244,9 @@ bool ExpectLiteralsEqual(const Literal& expected, const Literal& actual, case F64: match = ExpectLiteralsEqual(expected, actual, &multi_index, 0); break; + case C64: + match = ExpectLiteralsEqual(expected, actual, &multi_index, 0); + break; case TUPLE: { bool tuple_match = true; for (int i = 0; i < actual.tuple_literals_size(); ++i) { @@ -325,6 +337,9 @@ class NearComparator { case F64: ExpectLiteralsNear(expected, actual, 0); break; + case C64: + ExpectLiteralsNear(expected, actual, 0); + break; default: LOG(FATAL) << "Unsupported primitive type in near comparator: " << PrimitiveType_Name(expected.shape().element_type()) @@ -365,6 +380,19 @@ class NearComparator { } private: + template + bool NanMismatch(NativeT lhs, NativeT rhs) { + return std::isnan(lhs) != std::isnan(rhs); + } + + template + void ExpectNear(NativeT expected, NativeT actual, + const ::testing::Message& message) { + EXPECT_NEAR(expected, actual, error_.abs) + << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" + << message; + } + // EXPECTs that the two given scalar values are within the error bound. Keeps // track of how many mismatches have occurred to keep the size of the output // manageable. @@ -390,7 +418,7 @@ class NearComparator { "index %s abs_diff %f rel_err %f", LiteralTestUtil::MultiIndexAsString(multi_index_).c_str(), abs_diff, rel_err); - bool nan_mismatch = std::isnan(actual) != std::isnan(expected); + bool nan_mismatch = NanMismatch(expected, actual); bool mismatch = (nan_mismatch || (abs_diff >= error_.abs && rel_err >= error_.rel)); if (mismatch) { @@ -398,11 +426,12 @@ class NearComparator { abs_expected_miscompare_sum_ += std::abs(expected); const int64 kMaxFailures = 2; if (num_miscompares_ < kMaxFailures) { - EXPECT_NEAR(expected, actual, error_.abs) - << "mismatch at index " + ::testing::Message msg; + msg << "mismatch at index " << LiteralTestUtil::MultiIndexAsString(multi_index_) << " abs diff " << abs_diff << " rel err " << rel_err << " failure #" << num_miscompares_; + ExpectNear(expected, actual, msg); } else if (num_miscompares_ == kMaxFailures) { LOG(ERROR) << "reached max 'loud' failure count; silently proceeding..."; @@ -470,6 +499,23 @@ class NearComparator { std::vector max_abs_multi_index_; }; +template <> +bool NearComparator::NanMismatch(complex64 lhs, complex64 rhs) { + return std::isnan(lhs.real()) != std::isnan(rhs.real()) || + std::isnan(lhs.imag()) != std::isnan(rhs.imag()); +} + +template <> +void NearComparator::ExpectNear(complex64 expected, complex64 actual, + const ::testing::Message& message) { + EXPECT_NEAR(expected.real(), actual.real(), error_.abs) + << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" + << message; + EXPECT_NEAR(expected.imag(), actual.imag(), error_.abs) + << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" + << message; +} + } // namespace /* static */ ::testing::AssertionResult LiteralTestUtil::Near( diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h index ea8b4b7b98..3d78466107 100644 --- a/tensorflow/compiler/xla/types.h +++ b/tensorflow/compiler/xla/types.h @@ -35,6 +35,8 @@ using ::tensorflow::uint16; using ::tensorflow::uint32; using ::tensorflow::uint64; +typedef std::complex complex64; + using ::Eigen::half; } // namespace xla diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index eae284afb7..7ad61fab81 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -48,6 +48,9 @@ enum PrimitiveType { F32 = 11; F64 = 12; + // Complex values of fixed width. + C64 = 15; + // A tuple is a polymorphic sequence; e.g. a shape that holds different // sub-shapes. They are used for things like returning multiple values from a // computation; e.g. a computation that returns weights and biases may have a @@ -305,6 +308,7 @@ message LiteralProto { repeated uint64 u64s = 7; repeated float f32s = 8; repeated double f64s = 9; + repeated float c64s = 12; // Stored as interleaved real, imag floats. repeated LiteralProto tuple_literals = 10; bytes f16s = 11; // Note: the F16s are encoded in little endian byte order } -- GitLab From 555c63d173a41ceaba89513bb5f1b4ac9a4c86e4 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Mon, 23 Oct 2017 15:06:19 -0700 Subject: [PATCH 285/573] Automated g4 rollback of changelist 172946149 PiperOrigin-RevId: 173176850 --- tensorflow/compiler/aot/compile.cc | 6 +++--- tensorflow/compiler/aot/flags.cc | 5 +++-- tensorflow/compiler/aot/flags.h | 2 +- tensorflow/compiler/aot/tfcompile.bzl | 3 --- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc index 77c4ec88cb..eac8da0ab1 100644 --- a/tensorflow/compiler/aot/compile.cc +++ b/tensorflow/compiler/aot/compile.cc @@ -97,11 +97,11 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config, TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client, &computation, &compile_result->has_context_arg)); - if (!flags.out_session_module.empty()) { + if (!flags.debug_dir.empty()) { TF_ASSIGN_OR_RETURN(std::unique_ptr module, computation.Snapshot()); - TF_RETURN_IF_ERROR( - WriteBinaryProto(Env::Default(), flags.out_session_module, *module)); + string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb"); + TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module)); } xla::cpu::CpuAotCompilationOptions aot_opts( flags.target_triple, flags.target_cpu, flags.target_features, diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc index 7c2f27e550..5aff10346f 100644 --- a/tensorflow/compiler/aot/flags.cc +++ b/tensorflow/compiler/aot/flags.cc @@ -33,6 +33,9 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) { "fetch nodes will be dumped to stdout in a comma-separated list. " "Typically used to format arguments for other tools, e.g. " "freeze_graph."}, + {"debug_dir", &flags->debug_dir, + "Specifies a directory to dump debugging information, including " + "rewritten graphs and the XLA HLO module."}, // Flags controlling the XLA ahead-of-time compilation, that correspond to // the fields of xla::cpu::CpuAotCompilationOptions. // @@ -61,8 +64,6 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) { "namespaces are given, within the global namespace."}, {"out_object", &flags->out_object, "Output object file name."}, {"out_header", &flags->out_header, "Output header file name."}, - {"out_session_module", &flags->out_session_module, - "Output session module proto."}, {"gen_name_to_index", &flags->gen_name_to_index, "Generate name-to-index data for Lookup{Arg,Result}Index methods."}, {"gen_program_shape", &flags->gen_program_shape, diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h index 3519659e3a..3246dbf95c 100644 --- a/tensorflow/compiler/aot/flags.h +++ b/tensorflow/compiler/aot/flags.h @@ -29,6 +29,7 @@ struct MainFlags { string graph; string config; bool dump_fetch_nodes = false; + string debug_dir; string target_triple; string target_cpu; string target_features; @@ -36,7 +37,6 @@ struct MainFlags { string cpp_class; string out_object; string out_header; - string out_session_module; // C++ codegen options bool gen_name_to_index = false; diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl index 0ecfbedcb4..4888760acd 100644 --- a/tensorflow/compiler/aot/tfcompile.bzl +++ b/tensorflow/compiler/aot/tfcompile.bzl @@ -129,7 +129,6 @@ def tf_library(name, graph, config, # Rule that runs tfcompile to produce the header and object file. header_file = name + ".h" object_file = name + ".o" - session_module_pb = name + "_session_module.pb" ep = ("__" + PACKAGE_NAME + "__" + name).replace("/", "_") native.genrule( name=("gen_" + name), @@ -140,7 +139,6 @@ def tf_library(name, graph, config, outs=[ header_file, object_file, - session_module_pb, ], cmd=("$(location " + tfcompile_tool + ")" + " --graph=$(location " + tfcompile_graph + ")" + @@ -150,7 +148,6 @@ def tf_library(name, graph, config, " --target_triple=" + target_llvm_triple() + " --out_header=$(@D)/" + header_file + " --out_object=$(@D)/" + object_file + - " --out_session_module=$(@D)/" + session_module_pb + " " + (tfcompile_flags or "")), tools=[tfcompile_tool], visibility=visibility, -- GitLab From fbb71d767d890ead9007e713ae77dd223df232bd Mon Sep 17 00:00:00 2001 From: Sarah Maddox Date: Tue, 24 Oct 2017 09:20:44 +1100 Subject: [PATCH 286/573] Add links and fix typs Add links to the docs in GitHub, to make it easier for contributors to find them. Also fix some typos in the names of GitHub and TensorFlow, and standardise capitalisation in headings. --- .../docs_src/community/documentation.md | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md index 655506b098..77d4e0caec 100644 --- a/tensorflow/docs_src/community/documentation.md +++ b/tensorflow/docs_src/community/documentation.md @@ -1,6 +1,6 @@ # Writing TensorFlow Documentation -We welcome contributions to the Tensorflow documentation from the community. +We welcome contributions to the TensorFlow documentation from the community. This document explains how you can contribute to that documentation. In particular, this document explains the following: @@ -8,28 +8,30 @@ particular, this document explains the following: * How to make conformant edits. * How to build and test your documentation changes before you submit them. -You can view Tensorflow documentation on https://www.tensorflow.org, and you -can view and edit the raw files on Github. We're publishing our docs on Github -so everybody can contribute. Whatever gets checked in tensorflow/docs_src will -be published soon after on https://www.tensorflow.org. +You can view TensorFlow documentation on https://www.tensorflow.org, and you +can view and edit the raw files on +[GitHub](https://www.tensorflow.org/code/tensorflow/docs_src/). +We're publishing our docs on GitHub so everybody can contribute. Whatever gets +checked in to `tensorflow/docs_src` will be published soon after on +https://www.tensorflow.org. Republishing TensorFlow documentation in different forms is absolutely allowed, but we are unlikely to accept other documentation formats (or the tooling to generate them) into our repository. If you do choose to republish our documentation in another form, please be sure to include: -* The version of the API this represents (i.e. r1.0, master, etc.) +* The version of the API this represents (for example, r1.0, master, etc.) * The commit or version from which the documentation was generated * Where to get the latest documentation (that is, https://www.tensorflow.org) * The Apache 2.0 license. -## A Note on Versions +## A note on versions tensorflow.org, at root, shows documentation for the latest stable binary. This is the documentation you should be reading if you are using `pip` to install TensorFlow. -However, most developers will contribute documentation into the master Github +However, most developers will contribute documentation into the master GitHub branch, which is published, occasionally, at [tensorflow.org/versions/master](https://www.tensorflow.org/versions/master). @@ -49,8 +51,9 @@ in the code: To modify the reference documentation, you edit the appropriate code comments. Non-reference documentation (for example, the TensorFlow installation guides) is -authored by humans. This documentation is located in the `tensorflow/docs_src` -directory. Each subdirectory of `docs_src` contains a set of related Tensorflow +authored by humans. This documentation is located in the +[`tensorflow/docs_src`](https://www.tensorflow.org/code/tensorflow/docs_src/) +directory. Each subdirectory of `docs_src` contains a set of related TensorFlow documentation. For example, the TensorFlow installation guides are all in the `docs_src/install` directory. @@ -183,7 +186,7 @@ documentation in the `/tmp/tfdocs` dir: Note: You must set `src_dir` and `output_dir` to absolute file paths. -## Generating Python API Documentation +## Generating Python API documentation Ops, classes, and utility functions are defined in Python modules, such as `image_ops.py`. Python modules contain a module docstring. For example: @@ -216,7 +219,7 @@ the following: Only top level modules (currently just `tf` and `tfdbg`) need to be manually added to the generate script. -### Sealing Modules +### Sealing modules Because the doc generator walks all visible symbols, and descends into anything it finds, it will document any accidentally exposed symbols. If a module only @@ -242,7 +245,7 @@ following options for dealing with them: We'll discuss these options in detail below. -#### Private Symbols and Imports +#### Private symbols and imports The easiest way to conform to the API sealing expectations is to make non-public symbols private (by prepending an underscore _). The doc generator respects @@ -288,7 +291,7 @@ are public. All `@@`s will eventually be removed. If you see them, however, please do not randomly delete them as they are still in use by some of our systems. -#### Traversal Blacklist +#### Traversal blacklist If all else fails, you may add entries to the traversal blacklist in `generate_lib.py.` **Almost all entries in this list are an abuse of its @@ -311,7 +314,7 @@ flags, ...) included for platform abstraction can be documented without documenting their interior. Its use beyond this purpose is a shortcut that may be acceptable for contrib, but not for core tensorflow. -## Op Documentation Style Guide +## Op documentation style guide Long, descriptive module-level documentation for modules should go in the API Guides in `docs_src/api_guides/python`. @@ -334,7 +337,7 @@ is [here](https://daringfireball.net/projects/markdown/). You are allowed to use [MathJax](https://www.mathjax.org) notation for equations (see above for restrictions). -### Writing About Code +### Writing about code Put backticks around these things when they're used in text: @@ -375,7 +378,7 @@ Two notes about backticks for code samples in Markdown: However, do NOT indent four spaces and use backticks simultaneously. Use one or the other. -### Tensor Dimensions +### Tensor dimensions When you're talking about a tensor in general, don't capitalize the word tensor. When you're talking about the specific object that's provided to an op as an @@ -500,7 +503,7 @@ def foo(x, y, name="bar"): """ ``` -## Description of the Docstring Sections +## Description of the docstring sections This section details each of the elements in docstrings. -- GitLab From fd182dd02e431e7a7f16bd0ad1547405e591bc82 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 15:19:06 -0700 Subject: [PATCH 287/573] metrics.Mean writes a summary. PiperOrigin-RevId: 173178780 --- tensorflow/contrib/eager/python/BUILD | 1 + .../contrib/eager/python/metrics_impl.py | 5 +++- .../contrib/eager/python/metrics_test.py | 28 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index ace17424fe..a83012e17b 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -129,6 +129,7 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ + "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 2139c2b4b9..959ee735b0 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -20,6 +20,7 @@ from __future__ import print_function import re +from tensorflow.contrib.summary import summary_ops from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import dtypes @@ -235,7 +236,9 @@ class Mean(Metric): self.numer.assign_add(math_ops.reduce_sum(values)) def result(self): - return self.numer / self.denom + t = self.numer / self.denom + summary_ops.scalar(name=self.name, tensor=t) + return t class Accuracy(Mean): diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 9743666c89..1880e762d4 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -18,12 +18,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os +import tempfile + from tensorflow.contrib.eager.python import metrics +from tensorflow.contrib.summary import summary_ops +from tensorflow.core.util import event_pb2 from tensorflow.python.eager import context from tensorflow.python.eager import test from tensorflow.python.framework import dtypes +from tensorflow.python.lib.io import tf_record from tensorflow.python.ops import array_ops from tensorflow.python.ops import variables +from tensorflow.python.platform import gfile +from tensorflow.python.training import training_util class MetricsTest(test.TestCase): @@ -37,6 +45,26 @@ class MetricsTest(test.TestCase): self.assertEqual(dtypes.float64, m.dtype) self.assertEqual(dtypes.float64, m.result().dtype) + def testWriteSummaries(self): + m = metrics.Mean() + m([1, 10, 100]) + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name="t0").as_default(), summary_ops.always_record_summaries(): + m.result() # As a side-effect will write summaries. + + self.assertTrue(gfile.Exists(logdir)) + files = gfile.ListDirectory(logdir) + self.assertEqual(len(files), 1) + records = list( + tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) + self.assertEqual(len(records), 2) + event = event_pb2.Event() + event.ParseFromString(records[1]) + self.assertEqual(event.summary.value[0].simple_value, 37.0) + def testWeightedMean(self): m = metrics.Mean() m([1, 100, 100000], weights=[1, 0.2, 0.3]) -- GitLab From 57023e1b6c7d27e76d6f41d80b95402b9d93c467 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 15:59:39 -0700 Subject: [PATCH 288/573] tf.constant takes numpy dtypes in eager mode as well PiperOrigin-RevId: 173184568 --- tensorflow/python/eager/tensor_test.py | 5 +++++ tensorflow/python/framework/constant_op.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index e31c03c08d..b52bbe44d4 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.eager import context from tensorflow.python.eager import core from tensorflow.python.eager import test +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -102,6 +103,10 @@ class TFETensorTest(test_util.TensorFlowTestCase): t = _create_tensor(n) self.assertAllEqual([[1, 2], [3, 4]], t) + def testConstantDtype(self): + self.assertEqual(constant_op.constant(1.0, dtype=np.int64).dtype, + dtypes.int64) + def testTensorAndNumpyMatrix(self): expected = np.array([[1.0, 2.0], [3.0, 4.0]], np.float32) actual = _create_tensor([[1.0, 2.0], [3.0, 4.0]]) diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py index 34848af53b..d51e142da1 100644 --- a/tensorflow/python/framework/constant_op.py +++ b/tensorflow/python/framework/constant_op.py @@ -108,7 +108,10 @@ def convert_to_eager_tensor(value, ctx, dtype=None): dtype, value.dtype)) return value if dtype is not None: - dtype = dtype.as_datatype_enum + try: + dtype = dtype.as_datatype_enum + except AttributeError: + dtype = dtypes.as_dtype(dtype).as_datatype_enum device = ctx.device_name handle = ctx._handle # pylint: disable=protected-access if isinstance(value, (float,) + six.integer_types): -- GitLab From 6bcc00668094be8daf8465b8689bbed5ab285b2d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 16:24:08 -0700 Subject: [PATCH 289/573] Fix error in topk heap launch code. It assumed sizeof(struct) == sizeof(struct components). PiperOrigin-RevId: 173188044 --- tensorflow/core/kernels/topk_op_gpu.cu.cc | 2 +- tensorflow/python/kernel_tests/topk_op_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/topk_op_gpu.cu.cc b/tensorflow/core/kernels/topk_op_gpu.cu.cc index 10a7602dc4..ca296d5aa0 100644 --- a/tensorflow/core/kernels/topk_op_gpu.cu.cc +++ b/tensorflow/core/kernels/topk_op_gpu.cu.cc @@ -379,7 +379,7 @@ cudaError LaunchTopKKernel(const cudaStream_t& stream, int num_shards, // Use as many shards as possible. if (num_shards <= 0) { constexpr auto shared_memory_size = 48 << 10; // 48 KB - const auto heap_size = k * (sizeof(int) + sizeof(T)); + const auto heap_size = k * sizeof(Entry); // shared_memory_size = (num_shards + 1) * heap_size <=> num_shards = shared_memory_size / heap_size - 1; if (num_shards <= 0) { diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py index a8e7799cab..efb5b9f364 100644 --- a/tensorflow/python/kernel_tests/topk_op_test.py +++ b/tensorflow/python/kernel_tests/topk_op_test.py @@ -100,6 +100,13 @@ class TopKTest(test.TestCase): inputs = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.3, 0.4, 0.2]] self._validateTopK(inputs, 2, [[0.4, 0.3], [0.4, 0.3]], [[3, 1], [2, 1]]) + def testTop3(self): + k = 5 + inputs = np.random.permutation(np.linspace(0, 100, 6140, dtype=np.float64)) + indices = np.argsort(-inputs)[:k] + values = -np.sort(-inputs)[:k] + self._validateTopK(inputs, k, values, indices) + def _testLargeSort(self, dtype): b = 10 n = 5000 -- GitLab From ba1c7b8c67a09ea7af321860ef73a203bf27567a Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 16:31:11 -0700 Subject: [PATCH 290/573] tf.confusion_matrix works with eager execution enabled. PiperOrigin-RevId: 173188867 --- tensorflow/python/kernel_tests/confusion_matrix_test.py | 6 ++++-- tensorflow/python/ops/check_ops.py | 6 +++++- tensorflow/python/ops/control_flow_ops.py | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py index 2f56540a31..670a625f0f 100644 --- a/tensorflow/python/kernel_tests/confusion_matrix_test.py +++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import confusion_matrix from tensorflow.python.ops import math_ops @@ -32,6 +33,7 @@ from tensorflow.python.platform import test class ConfusionMatrixTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testExample(self): """This is a test of the example provided in pydoc.""" with self.test_session(): @@ -41,8 +43,8 @@ class ConfusionMatrixTest(test.TestCase): [0, 0, 1, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1] - ], confusion_matrix.confusion_matrix( - labels=[1, 2, 4], predictions=[2, 2, 4]).eval()) + ], self.evaluate(confusion_matrix.confusion_matrix( + labels=[1, 2, 4], predictions=[2, 2, 4]))) def _testConfMatrix(self, labels, predictions, truth, weights=None, num_classes=None): diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py index fb48175285..ceee009104 100644 --- a/tensorflow/python/ops/check_ops.py +++ b/tensorflow/python/ops/check_ops.py @@ -229,10 +229,14 @@ def assert_non_negative(x, data=None, summarize=None, message=None, name=None): with ops.name_scope(name, 'assert_non_negative', [x, data]): x = ops.convert_to_tensor(x, name='x') if data is None: + if context.in_eager_mode(): + name = str(x) + else: + name = x.name data = [ message, 'Condition x >= 0 did not hold element-wise:', - 'x (%s) = ' % x.name, x] + 'x (%s) = ' % name, x] zero = ops.convert_to_tensor(0, dtype=x.dtype) return assert_less_equal(zero, x, data=data, summarize=summarize) diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index 29aac913f0..f584d93aa2 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -116,6 +116,7 @@ def Assert(condition, data, summarize=None, name=None): Returns: assert_op: An `Operation` that, when executed, raises a `tf.errors.InvalidArgumentError` if `condition` is not true. + @compatibility{eager} returns None. """ with ops.name_scope(name, "Assert", [condition, data]) as name: xs = ops.convert_n_to_tensor(data) @@ -132,6 +133,8 @@ def Assert(condition, data, summarize=None, name=None): condition, data, summarize, name="Assert") guarded_assert = cond( condition, no_op, true_assert, name="AssertGuard") + if context.in_eager_mode(): + return return guarded_assert.op -- GitLab From 4f35cce3ed24455d30c46132ce3202db66009b31 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 16:52:56 -0700 Subject: [PATCH 291/573] Adds the ability to adjust the normalized value in batchnorm. By using random adjustments, it is possible to improve the model generalization. PiperOrigin-RevId: 173192179 --- tensorflow/python/layers/normalization.py | 96 ++++++++++++---- .../python/layers/normalization_test.py | 106 ++++++++++++++++++ ...nsorflow.layers.-batch-normalization.pbtxt | 2 +- .../tools/api/golden/tensorflow.layers.pbtxt | 2 +- 4 files changed, 180 insertions(+), 26 deletions(-) diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index df2b97f03e..74246189b5 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -101,6 +101,18 @@ class BatchNormalization(base.Layer): Normalization", which creates virtual sub-batches which are each normalized separately (with shared gamma, beta, and moving statistics). Must divide the actual batch size during execution. + adjustment: A function taking the `Tensor` containing the (dynamic) shape of + the input tensor and returning a pair (scale, bias) to apply to the + normalized values (before gamma and beta), only during training. For + example, if axis==-1, + `adjustment = lambda shape: ( + tf.random_uniform(shape[-1:], 0.93, 1.07), + tf.random_uniform(shape[-1:], -0.1, 0.1))` + will scale the normalized value by up to 7% up or down, then shift the + result by up to 0.1 (with independent scaling and bias for each feature + but shared across all examples), and finally apply gamma and/or beta. If + `None`, no adjustment is applied. Cannot be specified if + virtual_batch_size is specified. name: A string, the name of the layer. """ @@ -124,6 +136,7 @@ class BatchNormalization(base.Layer): fused=None, trainable=True, virtual_batch_size=None, + adjustment=None, name=None, **kwargs): super(BatchNormalization, self).__init__( @@ -143,6 +156,7 @@ class BatchNormalization(base.Layer): self.gamma_constraint = gamma_constraint self.renorm = renorm self.virtual_batch_size = virtual_batch_size + self.adjustment = adjustment if fused is None: fused = True @@ -192,6 +206,9 @@ class BatchNormalization(base.Layer): if 0 in self.axis: raise ValueError('When using virtual_batch_size, the batch dimension ' 'must be 0 and thus axis cannot include 0') + if self.adjustment is not None: + raise ValueError('When using virtual_batch_size, adjustment cannot ' + 'be specified') if self.fused: # Currently fused batch norm doesn't support renorm and beta/gamma @@ -204,7 +221,8 @@ class BatchNormalization(base.Layer): self.axis in [[1], [3]] and self.beta_regularizer is None and self.gamma_regularizer is None and - self.virtual_batch_size is None) + self.virtual_batch_size is None and + self.adjustment is None) # TODO(chrisying): fused batch norm is currently not supported for # multi-axis batch norm and by extension virtual batches. In some cases, # it might be possible to use fused batch norm but would require reshaping @@ -482,11 +500,41 @@ class BatchNormalization(base.Layer): if self.virtual_batch_size is not None: del reduction_axes[1] # Do not reduce along virtual batch dim - scale, offset = self.gamma, self.beta + # Broadcasting only necessary for single-axis batch norm where the axis is + # not the last dimension + broadcast_shape = [1] * ndims + broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value + def _broadcast(v): + if (v is not None and + len(v.get_shape()) != ndims and + reduction_axes != list(range(ndims - 1))): + return array_ops.reshape(v, broadcast_shape) + return v + + scale, offset = _broadcast(self.gamma), _broadcast(self.beta) + + def _compose_transforms(scale, offset, then_scale, then_offset): + if then_scale is not None: + scale *= then_scale + offset *= then_scale + if then_offset is not None: + offset += then_offset + return (scale, offset) # Determine a boolean value for `training`: could be True, False, or None. training_value = utils.constant_value(training) if training_value is not False: + if self.adjustment: + adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs)) + # Adjust only during training. + adj_scale = utils.smart_cond(training, + lambda: adj_scale, + lambda: array_ops.ones_like(adj_scale)) + adj_bias = utils.smart_cond(training, + lambda: adj_bias, + lambda: array_ops.zeros_like(adj_bias)) + scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset) + # Some of the computations here are not necessary when training==False # but not a constant. However, this makes the code simpler. keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 @@ -508,13 +556,9 @@ class BatchNormalization(base.Layer): # When training, the normalized values (say, x) will be transformed as # x * gamma + beta without renorm, and (x * r + d) * gamma + beta # = x * (r * gamma) + (d * gamma + beta) with renorm. - scale = array_ops.stop_gradient(r, name='renorm_r') - offset = array_ops.stop_gradient(d, name='renorm_d') - if self.gamma is not None: - scale *= self.gamma - offset *= self.gamma - if self.beta is not None: - offset += self.beta + r = _broadcast(array_ops.stop_gradient(r, name='renorm_r')) + d = _broadcast(array_ops.stop_gradient(d, name='renorm_d')) + scale, offset = _compose_transforms(r, d, scale, offset) else: new_mean, new_variance = mean, variance @@ -542,24 +586,14 @@ class BatchNormalization(base.Layer): else: mean, variance = self.moving_mean, self.moving_variance - # Broadcasting only necessary for single-axis batch norm where the axis is - # not the last dimension - broadcast_shape = [1] * ndims - broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value - rank = len(inputs.get_shape()) - def _broadcast(v): - if (v is not None and - len(v.get_shape()) != rank and - reduction_axes != list(range(ndims))[:-1]): - return array_ops.reshape(v, broadcast_shape) - return v - outputs = nn.batch_normalization(inputs, _broadcast(mean), _broadcast(variance), - _broadcast(offset), - _broadcast(scale), + offset, + scale, self.epsilon) + # If some components of the shape got lost due to adjustments, fix that. + outputs.set_shape(input_shape) if self.virtual_batch_size is not None: return undo_virtual_batching(outputs) @@ -589,7 +623,8 @@ def batch_normalization(inputs, renorm_clipping=None, renorm_momentum=0.99, fused=None, - virtual_batch_size=None): + virtual_batch_size=None, + adjustment=None): """Functional interface for the batch normalization layer. Reference: http://arxiv.org/abs/1502.03167 @@ -667,6 +702,18 @@ def batch_normalization(inputs, Normalization", which creates virtual sub-batches which are each normalized separately (with shared gamma, beta, and moving statistics). Must divide the actual batch size during execution. + adjustment: A function taking the `Tensor` containing the (dynamic) shape of + the input tensor and returning a pair (scale, bias) to apply to the + normalized values (before gamma and beta), only during training. For + example, if axis==-1, + `adjustment = lambda shape: ( + tf.random_uniform(shape[-1:], 0.93, 1.07), + tf.random_uniform(shape[-1:], -0.1, 0.1))` + will scale the normalized value by up to 7% up or down, then shift the + result by up to 0.1 (with independent scaling and bias for each feature + but shared across all examples), and finally apply gamma and/or beta. If + `None`, no adjustment is applied. Cannot be specified if + virtual_batch_size is specified. Returns: Output tensor. @@ -691,6 +738,7 @@ def batch_normalization(inputs, fused=fused, trainable=trainable, virtual_batch_size=virtual_batch_size, + adjustment=adjustment, name=name, _reuse=reuse, _scope=name) diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py index f8d9d2948c..90ebdc8c86 100644 --- a/tensorflow/python/layers/normalization_test.py +++ b/tensorflow/python/layers/normalization_test.py @@ -823,6 +823,112 @@ class BNTest(test.TestCase): self.assertAllClose(y_train, yt_val_train, atol=1e-5) self.assertAllClose(y_test, yt_val_test, atol=1e-5) + def testAdjustment(self): + shape = (4, 3) + xt = array_ops.placeholder(dtypes.float32, shape) + momentum = 0.99 + gamma = 2. + beta = 3. + epsilon = 0.001 + adjust_scale = random_ops.random_uniform(shape[-1:], 0.5, 1.5) + adjust_bias = random_ops.random_uniform(shape[-1:], -.2, .2) + bn = normalization_layers.BatchNormalization( + axis=1, + gamma_initializer=init_ops.constant_initializer(gamma), + beta_initializer=init_ops.constant_initializer(beta), + epsilon=epsilon, + momentum=momentum, + adjustment=lambda _: (adjust_scale, adjust_bias)) + training = array_ops.placeholder(dtypes.bool) + yt = bn.apply(xt, training=training) + + moving_mean = 0. + moving_variance = 1. + with self.test_session(use_gpu=True) as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(5): + x = np.random.random(shape) + yt_val_train, adj_scale_val, adj_bias_val = sess.run( + [yt, adjust_scale, adjust_bias] + bn.updates, + feed_dict={xt: x, training: True})[:3] + yt_val_test = sess.run([yt] + bn.updates, + feed_dict={xt: x, training: False})[0] + + mean = x.mean(0) + variance = x.var(0) + y_train = (((x - mean) / (variance + epsilon) ** 0.5) * adj_scale_val + + adj_bias_val) * gamma + beta + moving_mean += (mean - moving_mean) * (1. - momentum) + moving_variance += (variance - moving_variance) * (1. - momentum) + + y_test = ((x - moving_mean) / (moving_variance + epsilon) ** 0.5 * + gamma) + beta + + self.assertAllClose(y_train, yt_val_train, atol=1e-5) + self.assertAllClose(y_test, yt_val_test, atol=1e-5) + + def testRenormWithAdjustment(self): + shape = (4, 3) + xt = array_ops.placeholder(dtypes.float32, shape) + momentum = 0.99 + renorm_momentum = 0.8 + rmax = 1.1 + rmin = 0.9 + dmax = 0.1 + gamma = 2. + beta = 3. + epsilon = 0.001 + adjust_scale = random_ops.random_uniform(shape[-1:], 0.5, 1.5) + adjust_bias = random_ops.random_uniform(shape[-1:], -.2, .2) + bn = normalization_layers.BatchNormalization( + axis=1, + gamma_initializer=init_ops.constant_initializer(gamma), + beta_initializer=init_ops.constant_initializer(beta), + epsilon=epsilon, + momentum=momentum, + renorm=True, + renorm_clipping={'rmax': rmax, 'rmin': rmin, 'dmax': dmax}, + renorm_momentum=renorm_momentum, + adjustment=lambda _: (adjust_scale, adjust_bias)) + training = array_ops.placeholder(dtypes.bool) + yt = bn.apply(xt, training=training) + + moving_mean = 0. + moving_variance = 1. + renorm_mean = renorm_stddev = 0. + renorm_weight = 0. + with self.test_session(use_gpu=True) as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(5): + x = np.random.random(shape) + yt_val_train, adj_scale_val, adj_bias_val = sess.run( + [yt, adjust_scale, adjust_bias] + bn.updates, + feed_dict={xt: x, training: True})[:3] + yt_val_test = sess.run([yt] + bn.updates, + feed_dict={xt: x, training: False})[0] + + mean = x.mean(0) + stddev = np.sqrt(x.var(0) + epsilon) + adj_mean = renorm_mean + (1. - renorm_weight) * mean + adj_stddev = renorm_stddev + (1. - renorm_weight) * stddev + r = (stddev / adj_stddev).clip(rmin, rmax) + d = ((mean - adj_mean) / adj_stddev).clip(-dmax, dmax) + y_train = (((x - mean) / stddev * r + d) * adj_scale_val + + adj_bias_val) * gamma + beta + renorm_mean += (mean - renorm_mean) * (1. - renorm_momentum) + renorm_stddev += (stddev - renorm_stddev) * (1. - renorm_momentum) + renorm_weight += (1. - renorm_weight) * (1. - renorm_momentum) + moving_mean += (renorm_mean / renorm_weight - + moving_mean) * (1. - momentum) + moving_variance += ((renorm_stddev / renorm_weight) ** 2 - epsilon - + moving_variance) * (1. - momentum) + + y_test = ((x - moving_mean) / (moving_variance + epsilon) ** 0.5 * + gamma) + beta + + self.assertAllClose(y_train, yt_val_train, atol=1e-5) + self.assertAllClose(y_test, yt_val_test, atol=1e-5) + def testGhostBNNegativeVirtualBatch(self): shape = [6, 5, 4, 3] inp = random_ops.random_uniform(shape, seed=1) diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt index c66af13850..6e07b911a4 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt @@ -81,7 +81,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'adjustment\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt index dad514b534..c45d6e6c05 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt @@ -90,7 +90,7 @@ tf_module { } member_method { name: "batch_normalization" - argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\'], " + argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\', \'adjustment\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\', \'None\'], " } member_method { name: "conv1d" -- GitLab From ef3964f8ba27723c8db48af4f10801d5b7432db9 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Mon, 23 Oct 2017 17:04:42 -0700 Subject: [PATCH 292/573] Make BasicLSTMCell build its variables like a proper tf.layers.Layer. PiperOrigin-RevId: 173193748 --- tensorflow/python/ops/rnn_cell_impl.py | 108 ++++++++++++++---- ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt | 5 +- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index fb7b6d11a5..65b0407008 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -159,17 +159,17 @@ class RNNCell(base_layer.Layer): """Run this RNN cell on inputs, starting from the given state. Args: - inputs: `2-D` tensor with shape `[batch_size x input_size]`. + inputs: `2-D` tensor with shape `[batch_size, input_size]`. state: if `self.state_size` is an integer, this should be a `2-D Tensor` - with shape `[batch_size x self.state_size]`. Otherwise, if + with shape `[batch_size, self.state_size]`. Otherwise, if `self.state_size` is a tuple of integers, this should be a tuple - with shapes `[batch_size x s] for s in self.state_size`. + with shapes `[batch_size, s] for s in self.state_size`. scope: VariableScope for the created subgraph; defaults to class name. Returns: A pair containing: - - Output: A `2-D` tensor with shape `[batch_size x self.output_size]`. + - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. - New state: Either a single `2-D` tensor, or a tuple of tensors matching the arity and shapes of `state`. """ @@ -229,11 +229,11 @@ class RNNCell(base_layer.Layer): Returns: If `state_size` is an int or TensorShape, then the return value is a - `N-D` tensor of shape `[batch_size x state_size]` filled with zeros. + `N-D` tensor of shape `[batch_size, state_size]` filled with zeros. If `state_size` is a nested list or tuple, then the return value is a nested list or tuple (of the same structure) of `2-D` tensors with - the shapes `[batch_size x s]` for each s in `state_size`. + the shapes `[batch_size, s]` for each s in `state_size`. """ # Try to use the last cached zero_state. This is done to avoid recreating # zeros, especially when eager execution is enabled. @@ -285,6 +285,45 @@ class BasicRNNCell(RNNCell): return output, output +class _LayerRNNCell(RNNCell): + """Subclass of RNNCells that act like proper `tf.Layer` objects. + + For backwards compatibility purposes, most `RNNCell` instances allow their + `call` methods to instantiate variables via `tf.get_variable`. The underlying + variable scope thus keeps track of any variables, and returning cached + versions. This is atypical of `tf.layer` objects, which separate this + part of layer building into a `build` method that is only called once. + + Here we provide a subclass for `RNNCell` objects that act exactly as + `Layer` objects do. They must provide a `build` method and their + `call` methods do not access Variables `tf.get_variable`. + """ + + def __call__(self, inputs, state, scope=None): + """Run this RNN cell on inputs, starting from the given state. + + Args: + inputs: `2-D` tensor with shape `[batch_size, input_size]`. + state: if `self.state_size` is an integer, this should be a `2-D Tensor` + with shape `[batch_size, self.state_size]`. Otherwise, if + `self.state_size` is a tuple of integers, this should be a tuple + with shapes `[batch_size, s] for s in self.state_size`. + scope: `VariableScope` for the created subgraph; if not provided, + defaults to standard `tf.layers.Layer` behavior. + + Returns: + A pair containing: + + - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. + - New state: Either a single `2-D` tensor, or a tuple of tensors matching + the arity and shapes of `state`. + """ + # Bypass RNNCell's variable capturing semantics for LayerRNNCell. + # Instead, it is up to subclasses to provide a proper build + # method. See the class docstring for more details. + return base_layer.Layer.__call__(self, inputs, state, scope=scope) + + class GRUCell(RNNCell): """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). @@ -374,7 +413,7 @@ class LSTMStateTuple(_LSTMStateTuple): return c.dtype -class BasicLSTMCell(RNNCell): +class BasicLSTMCell(_LayerRNNCell): """Basic LSTM recurrent network cell. The implementation is based on: http://arxiv.org/abs/1409.2329. @@ -390,7 +429,7 @@ class BasicLSTMCell(RNNCell): """ def __init__(self, num_units, forget_bias=1.0, - state_is_tuple=True, activation=None, reuse=None): + state_is_tuple=True, activation=None, reuse=None, name=None): """Initialize the basic LSTM cell. Args: @@ -405,11 +444,14 @@ class BasicLSTMCell(RNNCell): reuse: (optional) Python boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. + name: String, the name of the layer. Layers with the same name will + share weights, but to avoid mistakes we require reuse=True in such + cases. When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. + `CudnnCompatibleLSTMCell` instead. """ - super(BasicLSTMCell, self).__init__(_reuse=reuse) + super(BasicLSTMCell, self).__init__(_reuse=reuse, name=name) if not state_is_tuple: logging.warn("%s: Using a concatenated state is slower and will soon be " "deprecated. Use state_is_tuple=True.", self) @@ -428,15 +470,35 @@ class BasicLSTMCell(RNNCell): def output_size(self): return self._num_units + def build(self, inputs_shape): + if inputs_shape.ndims != 2: + raise ValueError("Expected inputs.shape to be rank 2, saw shape: %s" + % inputs_shape) + if inputs_shape[1].value is None: + raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" + % inputs_shape) + + input_depth = inputs_shape[1].value + h_depth = self._num_units + self._kernel = self.add_variable( + _WEIGHTS_VARIABLE_NAME, + shape=[input_depth + h_depth, 4 * self._num_units]) + self._bias = self.add_variable( + _BIAS_VARIABLE_NAME, + shape=[4 * self._num_units], + initializer=init_ops.constant_initializer(0.0, dtype=self.dtype)) + + self._built = True + def call(self, inputs, state): """Long short-term memory cell (LSTM). Args: - inputs: `2-D` tensor with shape `[batch_size x input_size]`. + inputs: `2-D` tensor with shape `[batch_size, input_size]`. state: An `LSTMStateTuple` of state tensors, each shaped - `[batch_size x self.state_size]`, if `state_is_tuple` has been set to + `[batch_size, self.state_size]`, if `state_is_tuple` has been set to `True`. Otherwise, a `Tensor` shaped - `[batch_size x 2 * self.state_size]`. + `[batch_size, 2 * self.state_size]`. Returns: A pair containing the new hidden state, and the new state (either a @@ -451,11 +513,13 @@ class BasicLSTMCell(RNNCell): else: c, h = array_ops.split(value=state, num_or_size_splits=2, axis=one) - if self._linear is None: - self._linear = _Linear([inputs, h], 4 * self._num_units, True) + gate_inputs = math_ops.matmul( + array_ops.concat([inputs, h], 1), self._kernel) + gate_inputs = nn_ops.bias_add(gate_inputs, self._bias) + # i = input_gate, j = new_input, f = forget_gate, o = output_gate i, j, f, o = array_ops.split( - value=self._linear([inputs, h]), num_or_size_splits=4, axis=one) + value=gate_inputs, num_or_size_splits=4, axis=one) forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype) # Note that using `add` and `multiply` instead of `+` and `*` gives a @@ -585,16 +649,16 @@ class LSTMCell(RNNCell): """Run one step of LSTM. Args: - inputs: input Tensor, 2D, batch x num_units. + inputs: input Tensor, 2D, `[batch, num_units]. state: if `state_is_tuple` is False, this must be a state Tensor, - `2-D, batch x state_size`. If `state_is_tuple` is True, this must be a + `2-D, [batch, state_size]`. If `state_is_tuple` is True, this must be a tuple of state Tensors, both `2-D`, with column sizes `c_state` and `m_state`. Returns: A tuple containing: - - A `2-D, [batch x output_dim]`, Tensor representing the output of the + - A `2-D, [batch, output_dim]`, Tensor representing the output of the LSTM after reading `inputs` when previous state was `state`. Here output_dim is: num_proj if num_proj was set, @@ -1143,7 +1207,7 @@ class _Linear(object): """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. + args: a 2D Tensor or a list of 2D, batch, n, Tensors. output_size: int, second dimension of weight variable. dtype: data type for variables. build_bias: boolean, whether to build a bias variable. @@ -1225,7 +1289,7 @@ def _linear(args, """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. + args: a 2D Tensor or a list of 2D, batch, n, Tensors. output_size: int, second dimension of W[i]. bias: boolean, whether to add a bias term or not. bias_initializer: starting value to initialize the bias @@ -1233,7 +1297,7 @@ def _linear(args, kernel_initializer: starting value to initialize the weight. Returns: - A 2D Tensor with shape [batch x output_size] equal to + A 2D Tensor with shape `[batch, output_size]` equal to sum_i(args[i] * W[i]), where W[i]s are newly created matrices. Raises: diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt index 3254a62af1..b8e27cc6cb 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.nn.rnn_cell.BasicLSTMCell" tf_class { is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -90,7 +91,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'num_units\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\'], varargs=None, keywords=None, defaults=[\'1.0\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'num_units\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -110,7 +111,7 @@ tf_class { } member_method { name: "build" - argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None" } member_method { name: "call" -- GitLab From f1c6f5688de0a6fb3b8f016a7232c293b64689da Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 23 Oct 2017 17:07:22 -0700 Subject: [PATCH 293/573] EagerTensor.__copy__ and __deepcopy__ PiperOrigin-RevId: 173194115 --- tensorflow/python/eager/tensor_test.py | 12 ++++++++++++ tensorflow/python/framework/ops.py | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index b52bbe44d4..2b7b5c727a 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import copy + import numpy as np from tensorflow.python.eager import context @@ -103,6 +105,16 @@ class TFETensorTest(test_util.TensorFlowTestCase): t = _create_tensor(n) self.assertAllEqual([[1, 2], [3, 4]], t) + def testCopy(self): + t = constant_op.constant(1.0) + tt = copy.copy(t) + self.assertAllEqual(tt, 1.0) + del tt + tt = copy.deepcopy(t) + self.assertAllEqual(tt, 1.0) + del tt + self.assertAllEqual(t, 1.0) + def testConstantDtype(self): self.assertEqual(constant_op.constant(1.0, dtype=np.int64).dtype, dtypes.int64) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index b45cb2e0c6..b3caebce70 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -624,6 +624,15 @@ class _EagerTensorBase(Tensor): def _numpy(self): raise NotImplementedError() + def __copy__(self): + # Eager Tensors are immutable so it's safe to return themselves as a copy. + return self + + def __deepcopy__(self, memo): + # Eager Tensors are immutable so it's safe to return themselves as a copy. + del memo + return self + def _datatype_enum(self): raise NotImplementedError() -- GitLab From d75d5529d569d8f72cb215d3696db1feb1d9f033 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 23 Oct 2017 18:49:48 -0700 Subject: [PATCH 294/573] Revert to 64-bit indexing in extract_image_patches_op.h if the input/output tensor have more than 2^32 elements. PiperOrigin-RevId: 173203430 --- .../core/kernels/extract_image_patches_op.h | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/extract_image_patches_op.h b/tensorflow/core/kernels/extract_image_patches_op.h index 2abbed15e5..9d34daca64 100644 --- a/tensorflow/core/kernels/extract_image_patches_op.h +++ b/tensorflow/core/kernels/extract_image_patches_op.h @@ -32,11 +32,20 @@ struct ExtractImagePatchesForward { typename TTypes::Tensor output) { // Need to swap row/col when calling Eigen, because our data is in // NHWC format while Eigen assumes NWHC format. - To32Bit(output).device(d) = - To32Bit(input) - .extract_image_patches(patch_cols, patch_rows, stride_cols, - stride_rows, rate_cols, rate_rows, padding) - .reshape(output.dimensions()); + const int64 N = std::max(input.size(), output.size()); + if (N <= std::numeric_limits::max()) { + To32Bit(output).device(d) = + To32Bit(input) + .extract_image_patches(patch_cols, patch_rows, stride_cols, + stride_rows, rate_cols, rate_rows, padding) + .reshape(output.dimensions()); + } else { + output.device(d) = + input + .extract_image_patches(patch_cols, patch_rows, stride_cols, + stride_rows, rate_cols, rate_rows, padding) + .reshape(output.dimensions()); + } } }; -- GitLab From 1b46f888f28d67e52e4f40393d39410c74cfbb58 Mon Sep 17 00:00:00 2001 From: Sarah Maddox Date: Tue, 24 Oct 2017 14:01:39 +1100 Subject: [PATCH 295/573] Standardised caps on Virtualenv. --- tensorflow/docs_src/install/install_linux.md | 36 ++++++++++---------- tensorflow/docs_src/install/install_mac.md | 30 ++++++++-------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 9d204cc246..b641f403f5 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -77,22 +77,22 @@ TensorFlow with GPU support, but only if you do the following: You must pick the mechanism by which you install TensorFlow. The supported choices are as follows: - * [virtualenv](#InstallingVirtualenv) + * [Virtualenv](#InstallingVirtualenv) * ["native" pip](#InstallingNativePip) * [Docker](#InstallingDocker) * [Anaconda](#InstallingAnaconda) * installing from sources, which is documented in [a separate guide](https://www.tensorflow.org/install/install_sources). -**We recommend the virtualenv installation.** +**We recommend the Virtualenv installation.** [Virtualenv](https://virtualenv.pypa.io/en/stable/) is a virtual Python environment isolated from other Python development, incapable of interfering with or being affected by other Python programs -on the same machine. During the virtualenv installation process, +on the same machine. During the Virtualenv installation process, you will install not only TensorFlow but also all the packages that TensorFlow requires. (This is actually pretty easy.) To start working with TensorFlow, you simply need to "activate" the -virtual environment. All in all, virtualenv provides a safe and +virtual environment. All in all, Virtualenv provides a safe and reliable mechanism for installing and running TensorFlow. Native pip installs TensorFlow directly on your system without going @@ -121,26 +121,26 @@ Use that package at your own risk. -## Installing with virtualenv +## Installing with Virtualenv Take the following steps to install TensorFlow with Virtualenv: - 1. Install pip and virtualenv by issuing one of the following commands: + 1. Install pip and Virtualenv by issuing one of the following commands: 
$ sudo apt-get install python-pip python-dev python-virtualenv # for Python 2.7
      $ sudo apt-get install python3-pip python3-dev python-virtualenv # for Python 3.n
- 2. Create a virtualenv environment by issuing one of the following commands: + 2. Create a Virtualenv environment by issuing one of the following commands: 
$ virtualenv --system-site-packages targetDirectory # for Python 2.7
      $ virtualenv --system-site-packages -p python3 targetDirectory # for Python 3.n
where targetDirectory specifies the top of the - virtualenv tree. Our instructions assume that + Virtualenv tree. Our instructions assume that targetDirectory is `~/tensorflow`, but you may choose any directory. - 3. Activate the virtualenv environment by issuing one of the following + 3. Activate the Virtualenv environment by issuing one of the following commands: 
$ source ~/tensorflow/bin/activate # bash, sh, ksh, or zsh
@@ -156,18 +156,18 @@ Take the following steps to install TensorFlow with Virtualenv:
      
(tensorflow)$ easy_install -U pip

 
   5. Issue one of the following commands to install TensorFlow in the active
-     virtualenv environment:
+     Virtualenv environment:
 
      
(tensorflow)$ pip install --upgrade tensorflow      # for Python 2.7
      (tensorflow)$ pip3 install --upgrade tensorflow     # for Python 3.n
      (tensorflow)$ pip install --upgrade tensorflow-gpu  # for Python 2.7 and GPU
      (tensorflow)$ pip3 install --upgrade tensorflow-gpu # for Python 3.n and GPU

 
-     If the preceding command succeeds, skip Step 6. If the preceding
+     If the above command succeeds, skip Step 6. If the preceding
      command fails, perform Step 6.
 
   6. (Optional) If Step 5 failed (typically because you invoked a pip version
-     lower than 8.1), install TensorFlow in the active virtualenv environment
+     lower than 8.1), install TensorFlow in the active Virtualenv environment
      by issuing a command of the following format:
 
      
(tensorflow)$ pip install --upgrade tfBinaryURL   # Python 2.7
@@ -181,7 +181,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      [here](#the_url_of_the_tensorflow_python_package).  For example, if you
      are installing TensorFlow for Linux, Python 3.4, and CPU-only support,
      issue the following command to install TensorFlow in the active
-     virtualenv environment:
+     Virtualenv environment:
 
      
(tensorflow)$ pip3 install --upgrade \
      https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl

@@ -195,14 +195,14 @@ If you encounter installation problems, see
 After installing TensorFlow,
 [validate the installation](#ValidateYourInstallation).
 
-Note that you must activate the virtualenv environment each time you
-use TensorFlow. If the virtualenv environment is not currently active,
+Note that you must activate the Virtualenv environment each time you
+use TensorFlow. If the Virtualenv environment is not currently active,
 invoke one of the following commands:
 
 
$ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
 $ source ~/tensorflow/bin/activate.csh  # csh or tcsh

 
-When the virtualenv environment is active, you may run
+When the Virtualenv environment is active, you may run
 TensorFlow programs from this shell.  Your prompt will become
 the following to indicate that your tensorflow environment is active:
 
@@ -490,11 +490,11 @@ To validate your TensorFlow installation, do the following:
 
 ### Prepare your environment
 
-If you installed on native pip, virtualenv, or Anaconda, then
+If you installed on native pip, Virtualenv, or Anaconda, then
 do the following:
 
   1. Start a terminal.
-  2. If you installed with virtualenv or Anaconda, activate your container.
+  2. If you installed with Virtualenv or Anaconda, activate your container.
   3. If you installed TensorFlow source code, navigate to any
      directory *except* one containing TensorFlow source code.
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 6da22784bf..c95c27cd10 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -8,21 +8,21 @@ Note: As of version 1.2, TensorFlow no longer provides GPU support on macOS.
 
 You must pick the mechanism by which you install TensorFlow. The supported choices are as follows:
 
-  * virtualenv
+  * Virtualenv
   * "native" pip
   * Docker
   * installing from sources, which is documented in
     [a separate guide](https://www.tensorflow.org/install/install_sources).
 
-**We recommend the virtualenv installation.**
+**We recommend the Virtualenv installation.**
 [Virtualenv](https://virtualenv.pypa.io/en/stable)
 is a virtual Python environment isolated from other Python development,
 incapable of interfering with or being affected by other Python programs
-on the same machine.  During the virtualenv installation process,
+on the same machine.  During the Virtualenv installation process,
 you will install not only TensorFlow but also all the packages that
 TensorFlow requires.  (This is actually pretty easy.)
 To start working with TensorFlow, you simply need to "activate" the
-virtual environment.  All in all, virtualenv provides a safe and
+virtual environment.  All in all, Virtualenv provides a safe and
 reliable mechanism for installing and running TensorFlow.
 
 Native pip installs TensorFlow directly on your system without going through
@@ -48,30 +48,30 @@ However, within Anaconda, we recommend installing TensorFlow with the
 That is, the TensorFlow team neither tests nor maintains the conda package.
 Use that package at your own risk.
 
-## Installing with virtualenv
+## Installing with Virtualenv
 
 Take the following steps to install TensorFlow with Virtualenv:
 
   1. Start a terminal (a shell). You'll perform all subsequent steps
      in this shell.
 
-  2. Install pip and virtualenv by issuing the following commands:
+  2. Install pip and Virtualenv by issuing the following commands:
 
      
 $ sudo easy_install pip
      $ pip install --upgrade virtualenv 

 
-  3. Create a virtualenv environment by issuing a command of one
+  3. Create a Virtualenv environment by issuing a command of one
      of the following formats:
 
      
 $ virtualenv --system-site-packages targetDirectory # for Python 2.7
      $ virtualenv --system-site-packages -p python3 targetDirectory # for Python 3.n
      

 
-     where targetDirectory identifies the top of the virtualenv tree.
+     where targetDirectory identifies the top of the Virtualenv tree.
      Our instructions assume that targetDirectory
      is `~/tensorflow`, but you may choose any directory.
 
-  4. Activate the virtualenv environment by issuing one of the
+  4. Activate the Virtualenv environment by issuing one of the
      following commands:
 
      
$ source ~/tensorflow/bin/activate      # If using bash, sh, ksh, or zsh
@@ -93,7 +93,7 @@ Take the following steps to install TensorFlow with Virtualenv:
 
   7. Optional. If Step 6 failed (typically because you invoked a pip version
      lower than 8.1), install TensorFlow in the active
-     virtualenv environment by issuing a command of the following format:
+     Virtualenv environment by issuing a command of the following format:
 
      
 $ pip install --upgrade tfBinaryURL   # Python 2.7
      $ pip3 install --upgrade tfBinaryURL  # Python 3.n 

@@ -121,8 +121,8 @@ After installing TensorFlow,
 [validate your installation](#ValidateYourInstallation)
 to confirm that the installation worked properly.
 
-Note that you must activate the virtualenv environment each time you
-use TensorFlow in a new shell.  If the virtualenv environment is not
+Note that you must activate the Virtualenv environment each time you
+use TensorFlow in a new shell.  If the Virtualenv environment is not
 currently active (that is, the prompt is not `(tensorflow)`, invoke
 one of the following commands:
 
@@ -134,7 +134,7 @@ tensorflow environment is active:
 
 
 (tensorflow)$ 

 
-When the virtualenv environment is active, you may run
+When the Virtualenv environment is active, you may run
 TensorFlow programs from this shell.
 
 When you are done using TensorFlow, you may deactivate the
@@ -353,11 +353,11 @@ To validate your TensorFlow installation, do the following:
 
 ### Prepare your environment
 
-If you installed on native pip, virtualenv, or Anaconda, then
+If you installed on native pip, Virtualenv, or Anaconda, then
 do the following:
 
   1. Start a terminal.
-  2. If you installed with virtualenv or Anaconda, activate your container.
+  2. If you installed with Virtualenv or Anaconda, activate your container.
   3. If you installed TensorFlow source code, navigate to any
      directory *except* one containing TensorFlow source code.
 
-- 
GitLab


From 538c8ed28f3306ed724165b566c0a3d2ea817331 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Oct 2017 19:53:55 -0700
Subject: [PATCH 296/573] Updating gpu toolchain

PiperOrigin-RevId: 173207602
---
 third_party/toolchains/gpus/crosstool/CROSSTOOL | 2 +-
 third_party/toolchains/gpus/cuda/BUILD          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/third_party/toolchains/gpus/crosstool/CROSSTOOL b/third_party/toolchains/gpus/crosstool/CROSSTOOL
index 224b8912f6..a47e0c7cd7 100644
--- a/third_party/toolchains/gpus/crosstool/CROSSTOOL
+++ b/third_party/toolchains/gpus/crosstool/CROSSTOOL
@@ -296,7 +296,7 @@ toolchain {
   cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/5.4.0"
   cxx_builtin_include_directory: "/usr/include/c++/5.4.0/backward"
   cxx_builtin_include_directory: "/usr/local/include"
-  cxx_builtin_include_directory: "/usr/local/lib/clang/5.0.0/include"
+  cxx_builtin_include_directory: "/usr/local/lib/clang/6.0.0/include"
   cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
   cxx_builtin_include_directory: "/usr/include"
 }
diff --git a/third_party/toolchains/gpus/cuda/BUILD b/third_party/toolchains/gpus/cuda/BUILD
index 36be86cd10..39136de99c 100644
--- a/third_party/toolchains/gpus/cuda/BUILD
+++ b/third_party/toolchains/gpus/cuda/BUILD
@@ -1347,7 +1347,7 @@ genrule(
         "cuda/lib/libcupti.so.8.0",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart.so.8.0.61" "$(@D)/cuda/lib/libcudart.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcublas.so.8.0.71" "$(@D)/cuda/lib/libcublas.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcusolver.so.8.0.61" "$(@D)/cuda/lib/libcusolver.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcurand.so.8.0.61" "$(@D)/cuda/lib/libcurand.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcufft.so.8.0.61" "$(@D)/cuda/lib/libcufft.so.8.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.6.0.21" "$(@D)/cuda/lib/libcudnn.so.6" && cp "/usr/local/cuda-8.0/extras/CUPTI/lib64/libcupti.so.8.0.61" "$(@D)/cuda/lib/libcupti.so.8.0"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart.so.8.0.61" "$(@D)/cuda/lib/libcudart.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcublas.so.8.0.88" "$(@D)/cuda/lib/libcublas.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcusolver.so.8.0.61" "$(@D)/cuda/lib/libcusolver.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcurand.so.8.0.61" "$(@D)/cuda/lib/libcurand.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcufft.so.8.0.61" "$(@D)/cuda/lib/libcufft.so.8.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.6.0.21" "$(@D)/cuda/lib/libcudnn.so.6" && cp "/usr/local/cuda-8.0/extras/CUPTI/lib64/libcupti.so.8.0.61" "$(@D)/cuda/lib/libcupti.so.8.0"
    """,
 )
 
-- 
GitLab


From b20c66a2ad6055602b680ba8f7c8f359e104fd6b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Oct 2017 20:16:22 -0700
Subject: [PATCH 297/573] nsync update: portability fixes for MacOS, s390x.

PiperOrigin-RevId: 173208878
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 02540bd843..4d577fc246 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -429,11 +429,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "nsync",
       urls = [
-          "https://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
-          # "https://github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
+          "https://mirror.bazel.build/github.com/google/nsync/archive/839fcc53ff9be58218ed55397deb3f8376a1444e.tar.gz",
+          # "https://github.com/google/nsync/archive/839fcc53ff9be58218ed55397deb3f8376a1444e.tar.gz",
       ],
-      sha256 = "7dd8ca49319f77e8226cd020a9210a525f88ac26e7041c59c95418223a1cdf55",
-      strip_prefix = "nsync-ad722c76c6e6653f66be2e1f69521b7f7517da55",
+      sha256 = "124d105edb0313ef2d7f5bb86ec94d9f8de95479e55641c4254ffa8f795e9b37",
+      strip_prefix = "nsync-839fcc53ff9be58218ed55397deb3f8376a1444e",
   )
 
   native.http_archive(
-- 
GitLab


From 5bef42720aac651957139d78e4edf0f7bcda1a5f Mon Sep 17 00:00:00 2001
From: Anna R 
Date: Mon, 23 Oct 2017 21:53:28 -0700
Subject: [PATCH 298/573] Use := instead of ?= to set MAKEFILES_DIR to fix
 Android build flakiness

---
 tensorflow/contrib/makefile/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 525cf2cd41..c138fa0c1e 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -14,7 +14,9 @@
 # Host compilation settings
 
 # Find where we're running from, so we can store generated files here.
-MAKEFILE_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+ifeq ($(origin MAKEFILE_DIR), undefined)
+	MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+endif
 HAS_GEN_HOST_PROTOC := \
 $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\
 echo "false")
-- 
GitLab


From abbb80460f36f40641a42a03a04347143e2cc0ad Mon Sep 17 00:00:00 2001
From: Anna R 
Date: Mon, 23 Oct 2017 21:54:19 -0700
Subject: [PATCH 299/573] Internal change.

PiperOrigin-RevId: 173213868
---
 tensorflow/contrib/makefile/Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index cb23dd6dab..81024c26a4 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -14,7 +14,10 @@
 # Host compilation settings
 
 # Find where we're running from, so we can store generated files here.
-MAKEFILE_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+ifeq ($(origin MAKEFILE_DIR), undefined)
+	MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+endif
+
 HAS_GEN_HOST_PROTOC := \
 $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\
 echo "false")
-- 
GitLab


From b7de55e9ea79d1b6b1987834015b37ee59da5a99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Oct 2017 21:55:58 -0700
Subject: [PATCH 300/573] Handle non-Layer callables and "training" arguments
 in Sequential. Create "add()" method (analogous to Keras') now that
 add/track_layer() is inappropriate to use.

PiperOrigin-RevId: 173213945
---
 tensorflow/contrib/eager/python/BUILD         |  3 ++
 tensorflow/contrib/eager/python/network.py    | 42 +++++++++++++----
 .../contrib/eager/python/network_test.py      | 46 ++++++++++++++++++-
 3 files changed, 80 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index a83012e17b..3d7d307778 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -192,6 +192,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:layers_base",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python/estimator:util",
     ],
 )
 
@@ -203,6 +204,8 @@ py_test(
         ":network",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:layers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nn_ops",
         "//tensorflow/python/eager:test",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index 8ae5099546..28aed7628e 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -23,6 +23,7 @@ import uuid
 
 import six
 
+from tensorflow.python.estimator import util as estimator_util
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import base
 from tensorflow.python.ops import variable_scope
@@ -174,26 +175,47 @@ class Network(base.Layer):
 
 
 class Sequential(Network):
-  """Represents a linear sequence of Layers.
+  """Represents a linear sequence of Layers or functions.
 
-  The output of each layer is provided as the input to the next.
+  The output of each layer/function is provided as the input to the next.
   The inputs passed to `__call__` are passed to the inputs of the first
   Layer, and it returns the outputs of the last Layer.
 
   Args:
-    layers: An optional sequence of tf.layers.Layer objects.
+    layers_funcs: An optional sequence where each element is either a
+      tf.layers.Layer object or a callable.
     name: An optional string name to use for this Network.
   """
 
-  def __init__(self, layers=None, name=None):
+  def __init__(self, layers_funcs=None, name=None):
     super(Sequential, self).__init__(name=name)
-    if layers:
-      for l in layers:
-        self.track_layer(l)
+    self._layers_funcs = []
+    if layers_funcs:
+      for l in layers_funcs:
+        self.add(l)
+
+  def add(self, layer_func):
+    if isinstance(layer_func, base.Layer):
+      args = estimator_util.fn_args(layer_func.call)
+      self.track_layer(layer_func)
+    elif callable(layer_func):
+      args = estimator_util.fn_args(layer_func)
+    else:
+      raise TypeError(
+          "Sequential.add() takes only tf.layers.Layer objects or callables; "
+          "not '%s' of type '%s'." % (layer_func, type(layer_func)))
+    self._layers_funcs.append((("training" in args), layer_func))
 
-  def call(self, inputs):
+  def call(self, inputs, training=None):
     """Call each Layer in the order they were added."""
     # TODO(josh11b): Support "mode" and maybe other arguments
-    for l in self.layers:
-      inputs = l(inputs)
+    if training is None:
+      for _, l in self._layers_funcs:
+        inputs = l(inputs)
+    else:
+      for has_training_arg, l in self._layers_funcs:
+        if has_training_arg:
+          inputs = l(inputs, training)
+        else:
+          inputs = l(inputs)
     return inputs
diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py
index f43ce3acda..94cb73ae72 100644
--- a/tensorflow/contrib/eager/python/network_test.py
+++ b/tensorflow/contrib/eager/python/network_test.py
@@ -20,6 +20,8 @@ from tensorflow.contrib.eager.python import network
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.layers import core
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
 
 
 # pylint: disable=not-callable
@@ -94,7 +96,7 @@ class SequentialTest(test.TestCase):
 
     # Add a second layer to the network.
     l2 = core.Dense(1, use_bias=False)
-    net.track_layer(l2)
+    net.add(l2)
 
     # Set the second layer's weights so it multiplies by 11
     net(constant_op.constant([[2.0]]))  # Create l2's variables
@@ -102,6 +104,48 @@ class SequentialTest(test.TestCase):
     l2.trainable_variables[0].assign([[11.0]])
     self.assertEqual(231.0, net(constant_op.constant([[7.0]])).numpy())
 
+  def testFunctions(self):
+    # Create a sequential network with one function.
+    net = network.Sequential([nn_ops.relu])
+    two = constant_op.constant(2.0)
+    self.assertEqual(2.0, net(two).numpy())
+    self.assertEqual(0.0, net(-two).numpy())
+    # Add a second function.
+    net.add(math_ops.negative)
+    self.assertEqual(-2.0, net(two).numpy())
+
+  def testTrainingLayer(self):
+    net = network.Sequential([core.Dropout(0.99999)])
+    two = constant_op.constant(2.0)
+    self.assertEqual(2.0, net(two).numpy())
+    self.assertEqual(2.0, net(two, training=False).numpy())
+    for _ in range(20):
+      with_dropout = net(two, training=True).numpy()
+      self.assertIn(with_dropout, [0.0, 2.0])
+      if with_dropout == 0.0:
+        return
+    # Should only fail spuriously 1 in 10^100 runs.
+    self.fail("Didn't see dropout happen after 20 tries.")
+
+  def testTrainingFunction(self):
+    # Output depends on value of "training".
+    def add_training(input_value, training=None):
+      if training is None:
+        return input_value
+      elif training:
+        return input_value + 1
+      return input_value - 1
+
+    # Passing a "training" argument to double would cause an error.
+    def double(input_value):
+      return 2 * input_value
+
+    net = network.Sequential([add_training, double])
+    two = constant_op.constant(2)
+    self.assertEqual(4, net(two).numpy())
+    self.assertEqual(2, net(two, training=False).numpy())
+    self.assertEqual(6, net(two, training=True).numpy())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 20199e91b3503881ce9a4253d64fa783f731230f Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Mon, 23 Oct 2017 22:00:03 -0700
Subject: [PATCH 301/573] Don't prematurely return streams

PiperOrigin-RevId: 173214110
---
 tensorflow/compiler/xla/client/local_client.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index c885b815eb..15c744ecd3 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -175,10 +175,15 @@ StatusOr> LocalExecutable::Run(
   TF_RETURN_IF_ERROR(ValidateExecutionOptions(arguments, options, *backend_));
 
   ExecutableRunOptions actual_options = options;
+
+  Backend::StreamPtr stream;
   if (options.stream() == nullptr) {
+    // NB!  The lifetime of `stream` needs to match the lifetime of
+    // `actual_options` (otherwise we will end up using a returned stream in
+    // ExecuteOnStreamWrapper), which is why it isn't declared in the inner "if"
+    // scope.
     TF_ASSIGN_OR_RETURN(
-        Backend::StreamPtr stream,
-        BorrowStreamForDevice(options.device_ordinal(), backend_));
+        stream, BorrowStreamForDevice(options.device_ordinal(), backend_));
     actual_options.set_stream(stream.get());
   }
   if (options.allocator() == nullptr) {
-- 
GitLab


From 48591e00fe917bfeecc31e501ef133447b81e161 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 08:25:47 -0700
Subject: [PATCH 302/573] Better error message if you pass a list to
 tf.group().

PiperOrigin-RevId: 173260210
---
 tensorflow/python/ops/control_flow_ops.py      | 7 ++++++-
 tensorflow/python/ops/control_flow_ops_test.py | 9 +++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index f584d93aa2..dcdbeefb70 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -2918,7 +2918,6 @@ def group(*inputs, **kwargs):
 
   Args:
     *inputs: Zero or more tensors to group.
-    **kwargs: Optional parameters to pass when constructing the NodeDef.
     name: A name for this operation (optional).
 
   Returns:
@@ -2940,6 +2939,12 @@ def group(*inputs, **kwargs):
     # Sorts *inputs according to their devices.
     ops_on_device = {}  # device -> operations specified on the device.
     for inp in inputs:
+      if not hasattr(inp, "device"):
+        if isinstance(inp, list):
+          raise TypeError("To call tf.group() with a list, use "
+                          "tf.group(*[...]) not tf.group([...]).")
+        raise TypeError("Expected tf.group() expected Tensor arguments not "
+                        "'%s' with type '%s'" % (inp, type(inp)))
       dev = inp.device
       if dev in ops_on_device:
         ops_on_device[dev].append(inp)
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index d4e66ff1b3..34c405f293 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -114,6 +114,15 @@ class GroupTestCase(test_util.TensorFlowTestCase):
              device: "/task:2" }
     """, self._StripGraph(gd))
 
+  def testPassingList(self):
+    with ops.Graph().as_default():
+      a = constant_op.constant(0, name="a")
+      b = constant_op.constant(0, name="b")
+      with self.assertRaises(TypeError):
+        control_flow_ops.group([a.op, b.op])
+      with self.assertRaises(TypeError):
+        control_flow_ops.group(1, 2)
+
 
 class ShapeTestCase(test_util.TensorFlowTestCase):
 
-- 
GitLab


From c89ebd31d6d729704d77a712c7103f0a7e5353e1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 08:36:47 -0700
Subject: [PATCH 303/573] Creating a fix for a threading issue with ffmpeg_lib
 in third_party.

Bug is at:

#5804

Fix is to add a unique identifier to each temp file name. The id is unique to
the process. Multiple processes could still have a conflict, though even there
the odds do go down somewhat with this fix.

PiperOrigin-RevId: 173261202
---
 tensorflow/contrib/ffmpeg/default/BUILD       | 12 +++
 .../contrib/ffmpeg/default/ffmpeg_lib.cc      | 16 +++-
 .../ffmpeg/default/ffmpeg_lib_utility_test.cc | 80 +++++++++++++++++++
 3 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc

diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD
index 05fc658d80..949ae9ad9e 100644
--- a/tensorflow/contrib/ffmpeg/default/BUILD
+++ b/tensorflow/contrib/ffmpeg/default/BUILD
@@ -23,6 +23,18 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "ffmpeg_lib_utility_test",
+    srcs = ["ffmpeg_lib_utility_test.cc"],
+    deps = [
+        ":ffmpeg_lib",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_cc_test(
     name = "ffmpeg_lib_installed_test",
     srcs = ["ffmpeg_lib_test.cc"],
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
index b417a70b6e..545a4386d0 100644
--- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
@@ -198,6 +198,14 @@ string BuildWavFile(int32 samples_per_second, int32 channel_count,
   return data;
 }
 
+// Returns a unique number every time it is called.
+int64 UniqueId() {
+  static mutex mu(LINKER_INITIALIZED);
+  static int64 id = 0;
+  mutex_lock l(mu);
+  return ++id;
+}
+
 }  // namespace
 
 string GetTempFilename(const string& extension) {
@@ -208,8 +216,12 @@ string GetTempFilename(const string& extension) {
     }
     struct stat statbuf;
     if (!stat(dir, &statbuf) && S_ISDIR(statbuf.st_mode)) {
-      string tmp_filepath =
-          io::JoinPath(dir, StrCat("tmp_file_XXXXXX", ".", extension));
+      // UniqueId is added here because mkstemps is not as thread safe as it
+      // looks. https://github.com/tensorflow/tensorflow/issues/5804 shows
+      // the problem.
+      string tmp_filepath = io::JoinPath(
+          dir,
+          StrCat("tmp_file_tensorflow_", UniqueId(), "_XXXXXX.", extension));
       int fd = mkstemps(&tmp_filepath[0], extension.length() + 1);
       if (fd < 0) {
         LOG(FATAL) << "Failed to create temp file.";
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc
new file mode 100644
index 0000000000..7176f3b550
--- /dev/null
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc
@@ -0,0 +1,80 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace ffmpeg {
+namespace {
+
+TEST(FfmpegLibTest, TestTempDirectoryThreading) {
+  // Testing a fix for a bug that allowed different threads to create
+  // conflicting temp files.
+  // See github.com/tensorflow/tensorflow/issues/5804 for details.
+  const int32 kNumThreads = 10;
+  const int32 kNumWorkItems = 10000;
+  static constexpr size_t kStringsPerItem = 100;
+  Env* environment = Env::Default();
+  thread::ThreadPool pool(environment, "test", kNumThreads);
+
+  mutex mu;
+  std::vector temp_filenames;
+  temp_filenames.reserve(kNumWorkItems * kStringsPerItem);
+
+  // Queue a large number of work items for the threads to process. Each work
+  // item creates a temp file and then deletes it.
+  for (int i = 0; i < kNumWorkItems; ++i) {
+    pool.Schedule([&mu, &temp_filenames, environment]() {
+      std::array buffer;
+      for (int32 j = 0; j < kStringsPerItem; ++j) {
+        buffer[j] = GetTempFilename("mp3");
+        TF_QCHECK_OK(environment->DeleteFile(buffer[j]));
+      }
+      mutex_lock l(mu);
+      for (const auto& fn : buffer) {
+        temp_filenames.push_back(fn);
+      }
+    });
+  }
+
+  // Wait until all work items are complete.
+  while (true) {
+    mutex_lock l(mu);
+    if (temp_filenames.size() == kNumWorkItems * kStringsPerItem) {
+      break;
+    }
+  }
+
+  // Check that no duplicates are created.
+  std::set unique_filenames;
+  mutex_lock l(mu);
+  for (const auto& fn : temp_filenames) {
+    ASSERT_TRUE(unique_filenames.insert(fn).second);
+  }
+}
+
+}  // namespace
+}  // namespace ffmpeg
+}  // namespace tensorflow
-- 
GitLab


From d312011c0220affe5fc8ec5c0d0d61a605c79ed7 Mon Sep 17 00:00:00 2001
From: Chris Leary 
Date: Tue, 24 Oct 2017 08:59:50 -0700
Subject: [PATCH 304/573] [XLA] Elide whitespace in symbols in
 BatchNormRewriter.

PiperOrigin-RevId: 173263867
---
 tensorflow/compiler/xla/service/batchnorm_rewriter.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc
index 427294dfc6..abe881cd1a 100644
--- a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc
+++ b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc
@@ -83,11 +83,11 @@ class BatchNormRewriterVisitor : public DfsHloVisitorWithDefault {
 
   HloComputation* GetScalarBinaryComputation(PrimitiveType primitive_type,
                                              HloOpcode opcode) {
-    HloComputation::Builder b("scalar computation");
+    HloComputation::Builder b("scalar_computation");
     auto scalar_lhs = b.AddInstruction(HloInstruction::CreateParameter(
-        0, ShapeUtil::MakeShape(F32, {}), "scalar lhs"));
+        0, ShapeUtil::MakeShape(F32, {}), "scalar_lhs"));
     auto scalar_rhs = b.AddInstruction(HloInstruction::CreateParameter(
-        1, ShapeUtil::MakeShape(F32, {}), "scalar rhs"));
+        1, ShapeUtil::MakeShape(F32, {}), "scalar_rhs"));
     auto scalar_op = b.AddInstruction(
         HloInstruction::CreateBinary(ShapeUtil::MakeShape(primitive_type, {}),
                                      opcode, scalar_lhs, scalar_rhs));
-- 
GitLab


From 86895d4a87a4d2cf2e1106b3fa3c176378d1029a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 09:22:56 -0700
Subject: [PATCH 305/573] Provide better debug information on true_classes
 assertion

PiperOrigin-RevId: 173266690
---
 tensorflow/core/kernels/candidate_sampler_ops.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/candidate_sampler_ops.cc b/tensorflow/core/kernels/candidate_sampler_ops.cc
index 9e8b122801..e937c4f11b 100644
--- a/tensorflow/core/kernels/candidate_sampler_ops.cc
+++ b/tensorflow/core/kernels/candidate_sampler_ops.cc
@@ -44,9 +44,11 @@ class BaseCandidateSamplerOp : public OpKernel {
     OP_REQUIRES(context, true_classes.dims() == 2,
                 errors::InvalidArgument("true_classes must be a matrix"));
     const int32 batch_size = true_classes.dim_size(0);
-    OP_REQUIRES(context, true_classes.dim_size(1) == num_true_,
-                errors::InvalidArgument("true_classes must have "
-                                        "num_true columns"));
+    OP_REQUIRES(
+        context, true_classes.dim_size(1) == num_true_,
+        errors::InvalidArgument("true_classes must have "
+                                "num_true columns, expected: ",
+                                true_classes.dim_size(1), " was: ", num_true_));
     CHECK(sampler_) << "CandidateSamplerOp did not set sampler_";
 
     if (unique_) {
-- 
GitLab


From 58b071639d97afdbc5ac5e222a4be81dcb344962 Mon Sep 17 00:00:00 2001
From: Katherine Wu 
Date: Tue, 24 Oct 2017 10:08:03 -0700
Subject: [PATCH 306/573] Added a dataset page to the api guide

PiperOrigin-RevId: 173272637
---
 .../api_guides/python/input_dataset.md        | 81 +++++++++++++++++++
 .../api_guides/python/reading_data.md         | 23 ++++--
 .../docs_src/programmers_guide/datasets.md    |  2 +-
 3 files changed, 98 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/docs_src/api_guides/python/input_dataset.md

diff --git a/tensorflow/docs_src/api_guides/python/input_dataset.md b/tensorflow/docs_src/api_guides/python/input_dataset.md
new file mode 100644
index 0000000000..2798d76be9
--- /dev/null
+++ b/tensorflow/docs_src/api_guides/python/input_dataset.md
@@ -0,0 +1,81 @@
+# `Dataset` Input Pipeline
+[TOC]
+
+@{tf.data.Dataset} allows you to build complex input pipelines. See the
+@{$datasets$programmer's guide} for an in-depth explanation of how to use this
+API.
+
+## Reader classes
+
+Classes that create a dataset from input files.
+
+*   @{tf.data.FixedLengthRecordDataset}
+*   @{tf.data.TextLineDataset}
+*   @{tf.data.TFRecordDataset}
+
+## Creating new datasets
+
+Static methods in `Dataset` that create new datasets.
+
+*   @{tf.data.Dataset.from_generator}
+*   @{tf.data.Dataset.from_sparse_tensor_slices}
+*   @{tf.data.Dataset.from_tensor_slices}
+*   @{tf.data.Dataset.from_tensors}
+*   @{tf.data.Dataset.list_files}
+*   @{tf.data.Dataset.range}
+*   @{tf.data.Dataset.zip}
+
+## Transformations on existing datasets
+
+These functions transform an existing dataset, and return a new dataset. Calls
+can be chained together, as shown in the example below:
+
+```
+train_data = train_data.batch(100).shuffle().repeat()
+```
+
+*   @{tf.data.Dataset.apply}
+*   @{tf.data.Dataset.batch}
+*   @{tf.data.Dataset.cache}
+*   @{tf.data.Dataset.concatenate}
+*   @{tf.data.Dataset.filter}
+*   @{tf.data.Dataset.flat_map}
+*   @{tf.data.Dataset.interleave}
+*   @{tf.data.Dataset.map}
+*   @{tf.data.Dataset.padded_batch}
+*   @{tf.data.Dataset.prefetch}
+*   @{tf.data.Dataset.repeat}
+*   @{tf.data.Dataset.shard}
+*   @{tf.data.Dataset.shuffle}
+*   @{tf.data.Dataset.skip}
+*   @{tf.data.Dataset.take}
+
+### Custom transformation functions
+
+Custom transformation functions can be applied to a `Dataset` using @{tf.data.Dataset.apply}. Below are custom transformation functions from `tf.contrib.data`:
+
+*   @{tf.contrib.data.batch_and_drop_remainder}
+*   @{tf.contrib.data.dense_to_sparse_batch}
+*   @{tf.contrib.data.enumerate_dataset}
+*   @{tf.contrib.data.group_by_window}
+*   @{tf.contrib.data.ignore_errors}
+*   @{tf.contrib.data.rejection_resample}
+*   @{tf.contrib.data.sloppy_interleave}
+*   @{tf.contrib.data.unbatch}
+
+## Iterating over datasets
+
+These functions make a @{tf.data.Iterator} from a `Dataset`.
+
+*   @{tf.data.Dataset.make_initializable_iterator}
+*   @{tf.data.Dataset.make_one_shot_iterator}
+
+The `Iterator` class also contains static methods that create a @{tf.data.Iterator} that can be used with multiple `Dataset` objects.
+
+*   @{tf.data.Iterator.from_structure}
+*   @{tf.data.Iterator.from_string_handle}
+
+## Extra functions from `tf.contrib.data`
+
+*   @{tf.contrib.data.read_batch_features}
+
diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md
index 8b6196ea34..7609ca91d0 100644
--- a/tensorflow/docs_src/api_guides/python/reading_data.md
+++ b/tensorflow/docs_src/api_guides/python/reading_data.md
@@ -3,16 +3,25 @@
 Note: The preferred way to feed data into a tensorflow program is using the
 @{$datasets$Datasets API}.
 
-There are three other methods of getting data into a TensorFlow program:
+There are four methods of getting data into a TensorFlow program:
 
+*   `Dataset` API: Easily construct a complex input pipeline. (preferred method)
 *   Feeding: Python code provides the data when running each step.
-*   Reading from files: an input pipeline reads the data from files
+*   `QueueRunner`: a queue-based input pipeline reads the data from files
     at the beginning of a TensorFlow graph.
 *   Preloaded data: a constant or variable in the TensorFlow graph holds
     all the data (for small data sets).
 
 [TOC]
 
+## Dataset API
+
+See the @{$datasets$programmer's guide} for an in-depth explanation of
+@{tf.data.Dataset}. The `Dataset` API allows you to extract and preprocess data
+from different input/file formats, and apply transformations such as batch,
+shuffle, and map to the dataset. This is an improved version of the old input
+methods, feeding and `QueueRunner`.
+
 ## Feeding
 
 TensorFlow's feed mechanism lets you inject data into any Tensor in a
@@ -22,7 +31,7 @@ graph.
 Supply feed data through the `feed_dict` argument to a run() or eval() call
 that initiates computation.
 
-Note: "Feeding" is the least efficient way to feed data into a tensorflow
+Warning: "Feeding" is the least efficient way to feed data into a tensorflow
 program and should only be used for small experiments and debugging.
 
 ```python
@@ -44,9 +53,9 @@ in
 [`tensorflow/examples/tutorials/mnist/fully_connected_feed.py`](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/fully_connected_feed.py),
 and is described in the @{$mechanics$MNIST tutorial}.
 
-## Reading from files
+## `QueueRunner`
 
-A typical pipeline for reading records from files has the following stages:
+A typical queue-based pipeline for reading records from files has the following stages:
 
 1.  The list of filenames
 2.  *Optional* filename shuffling
@@ -57,8 +66,8 @@ A typical pipeline for reading records from files has the following stages:
 7.  *Optional* preprocessing
 8.  Example queue
 
-Note: This section discusses implementing input pipelines using the
-queue-based APIs which can be cleanly replaced by the ${$datasets$Dataset API}.
+Warning: This section discusses implementing input pipelines using the
+queue-based APIs which can be cleanly replaced by the @{$datasets$Dataset API}.
 
 ### Filenames, shuffling, and epoch limits
 
diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md
index fd1c927539..38e5612fb4 100644
--- a/tensorflow/docs_src/programmers_guide/datasets.md
+++ b/tensorflow/docs_src/programmers_guide/datasets.md
@@ -1,6 +1,6 @@
 # Importing Data
 
-The `Dataset` API enables you to build complex input pipelines from
+The @{tf.data.Dataset$`Dataset`} API enables you to build complex input pipelines from
 simple, reusable pieces. For example, the pipeline for an image model might
 aggregate data from files in a distributed file system, apply random
 perturbations to each image, and merge randomly selected images into a batch
-- 
GitLab


From 377dd3d0d51f93f22eadfd18f4186c27d8506d69 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 10:11:42 -0700
Subject: [PATCH 307/573] Use tf.where instead of multiplies when masking
 probabilities in the BeamSearchDecoder.

PiperOrigin-RevId: 173273139
---
 .../seq2seq/python/ops/beam_search_decoder.py     | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index a88d4f5b8b..5be0c92243 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -715,12 +715,6 @@ def _mask_probs(probs, eos_token, finished):
     probability on the EOS token.
   """
   vocab_size = array_ops.shape(probs)[2]
-  finished_mask = math_ops.cast(array_ops.expand_dims(finished, 2), probs.dtype)
-  not_finished_mask = math_ops.cast(
-      array_ops.expand_dims(math_ops.logical_not(finished), 2),
-      probs.dtype)
-  # These examples are not finished and we leave them
-  non_finished_examples = not_finished_mask * probs
   # All finished examples are replaced with a vector that has all
   # probability on EOS
   finished_row = array_ops.one_hot(
@@ -729,8 +723,13 @@ def _mask_probs(probs, eos_token, finished):
       dtype=probs.dtype,
       on_value=0.,
       off_value=probs.dtype.min)
-  finished_examples = finished_mask * finished_row
-  return finished_examples + non_finished_examples
+  finished_probs = array_ops.tile(
+      array_ops.reshape(finished_row, [1, 1, -1]),
+      array_ops.concat([array_ops.shape(finished), [1]], 0))
+  finished_mask = array_ops.tile(
+      array_ops.expand_dims(finished, 2), [1, 1, vocab_size])
+
+  return array_ops.where(finished_mask, finished_probs, probs)
 
 
 def _maybe_tensor_gather_helper(gather_indices, gather_from, batch_size,
-- 
GitLab


From 1bbec9e4e9c5d3fbbc2fa2b58841435e86dbf76a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 10:39:16 -0700
Subject: [PATCH 308/573] * Add GPU implementation of LogDeterminant op. *
 Switch GPU implementation of Determinant to use the more numerically stable
 kernel as well. * Change behavior for Determinant on matrices with
 (numerically) infinite determinants to match the behavior of
 numpy.linalg.det: Return inf for matrix with infinite determinant. * Misc.
 cleanup in code working around missing support for complex in the NVCC
 compiler.

PiperOrigin-RevId: 173277377
---
 tensorflow/core/kernels/cuda_solvers.h        |  10 -
 .../core/kernels/cuda_solvers_gpu.cu.cc       | 156 +-----------
 tensorflow/core/kernels/determinant_op.cc     | 224 ++++++++++++++----
 tensorflow/core/kernels/determinant_op.h      |  47 ++++
 .../core/kernels/determinant_op_gpu.cu.cc     | 168 +++++++++++++
 tensorflow/core/kernels/linalg_ops_common.h   |  15 +-
 tensorflow/core/kernels/matrix_inverse_op.cc  |   1 -
 tensorflow/core/kernels/matrix_solve_op.cc    |   9 +-
 .../kernels/matrix_triangular_solve_op.cc     |  10 +-
 .../kernel_tests/determinant_op_test.py       |   5 +-
 10 files changed, 414 insertions(+), 231 deletions(-)
 create mode 100644 tensorflow/core/kernels/determinant_op.h
 create mode 100644 tensorflow/core/kernels/determinant_op_gpu.cu.cc

diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h
index eb720b191f..af27eb6c47 100644
--- a/tensorflow/core/kernels/cuda_solvers.h
+++ b/tensorflow/core/kernels/cuda_solvers.h
@@ -410,16 +410,6 @@ class DeviceLapackInfo : public ScratchSpace {
 
 namespace functor {
 
-// Helper functor to compute the product of diagonal elements in all matrices
-// in a flattened batch.
-template 
-struct DeterminantFromPivotedLUFunctor {
-  void operator()(const Device& device,
-                  typename TTypes::ConstTensor lu_factor,
-                  const int* pivots, typename TTypes::Tensor output,
-                  int* info);
-};
-
 // Helper functor to set a batch of matrices to the identity.
 // TODO(rmlarsen): Use this kernel to replace the horribly inefficient tf.eye
 // op.
diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
index 4171f9d68e..84330c041a 100644
--- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
+++ b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
@@ -29,159 +29,11 @@ namespace functor {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-namespace {
-
-// Hacks around missing support for complex arithmetic in nvcc.
-template 
-__device__ inline Scalar Multiply(Scalar x, Scalar y) {
-  return x * y;
-}
-
-template <>
-__device__ inline cuComplex Multiply(cuComplex x, cuComplex y) {
-  return cuCmulf(x, y);
-}
-
-template <>
-__device__ inline cuDoubleComplex Multiply(cuDoubleComplex x,
-                                           cuDoubleComplex y) {
-  return cuCmul(x, y);
-}
-
-template 
-__device__ inline Scalar Negate(Scalar x) {
-  return -x;
-}
-
-template <>
-__device__ inline cuComplex Negate(cuComplex x) {
-  return make_cuComplex(-cuCrealf(x), -cuCimagf(x));
-}
-
-template <>
-__device__ inline cuDoubleComplex Negate(cuDoubleComplex x) {
-  return make_cuDoubleComplex(-cuCreal(x), -cuCimag(x));
-}
-
-template 
-__device__ inline bool IsFinite(Scalar x) {
-  return Eigen::numext::isfinite(x);
-}
-
-template <>
-__device__ inline bool IsFinite(cuComplex x) {
-  return Eigen::numext::isfinite(cuCrealf(x)) &&
-         Eigen::numext::isfinite(cuCimagf(x));
-}
-
-template <>
-__device__ inline bool IsFinite(cuDoubleComplex x) {
-  return Eigen::numext::isfinite(cuCreal(x)) &&
-         Eigen::numext::isfinite(cuCimag(x));
-}
-
-template 
-struct Const {
-  template 
-  __device__ static inline Scalar make_const(const RealScalar x) {
-    return Scalar(x);
-  }
-};
-
-template <>
-struct Const {
-  template 
-  __device__ static inline cuComplex make_const(const RealScalar x) {
-    return make_cuComplex(x, 0.0f);
-  }
-};
-
-template <>
-struct Const {
-  template 
-  __device__ static inline cuDoubleComplex make_const(const RealScalar x) {
-    return make_cuDoubleComplex(x, 0.0f);
-  }
-};
-
-}  // namespace
-
-template 
-__global__ void DeterminantFromPivotedLUKernel(int nthreads, int n,
-                                               const Scalar* lu_factor,
-                                               const int* all_pivots,
-                                               Scalar* dst, int* info) {
-  const int matrix_size = n * n;
-  const int stride = n + 1;
-  // We only parallelize over batches here. Performance is not critical,
-  // since this cheap O(n) kernel always follows an O(n^3) LU factorization.
-  // The main purpose is to avoid having to copy the LU decomposition to
-  // host memory.
-  CUDA_1D_KERNEL_LOOP(o_idx, nthreads) {
-    // Compute the order of the permutation from the number of transpositions
-    // encoded in the pivot array, see:
-    // http://icl.cs.utk.edu/lapack-forum/viewtopic.php?f=2&t=340
-    const int* pivots = all_pivots + o_idx * n;
-    int order = 0;
-    for (int i = 0; i < n - 1; ++i) {
-      // Notice: Internally, the cuBlas code uses Fortran convention (1-based)
-      // indexing so we expect pivots[i] == i + 1 for rows that were not moved.
-      order += pivots[i] != (i + 1);
-    }
-
-    // Compute the product of the diagonal elements of U from the partially
-    // pivoted LU factorization.
-    // TODO(rmlarsen): This naive implementation (matching that in Eigen used
-    // for the CPU kernel) is pathetically unstable. Should we implement
-    // log-determinant instead (a different set of ops altogether) or something
-    // like the method used in the old LINPACK code:
-    // http://www.netlib.org/linpack/dgedi.f ?
-    int i_idx = matrix_size * o_idx;
-    Scalar prod = lu_factor[i_idx];
-    for (int i = 1; i < n; ++i) {
-      i_idx += stride;
-      prod = Multiply(prod, lu_factor[i_idx]);
-    }
-    // Finally set the determinant to (-1)^order * prod(diag(U)).
-    dst[o_idx] = order % 2 ? Negate(prod) : prod;
-
-    // We write a magic value into the info array if the result was infinite.
-    if (!IsFinite(prod)) {
-      info[o_idx] = kint32min;
-    }
-  }
-}
-
-template 
-struct DeterminantFromPivotedLUFunctor {
-  void operator()(const GPUDevice& device,
-                  typename TTypes::ConstTensor lu_factor,
-                  const int* pivots, typename TTypes::Tensor output,
-                  int* info) {
-    using CudaType = typename CUDAComplexT::type;
-    const int64 num_matrices = output.size();
-    const int64 n = lu_factor.dimension(2);
-    const CudaType* lu_factor_ptr =
-        reinterpret_cast(lu_factor.data());
-    CudaType* output_ptr = reinterpret_cast(output.data());
-    CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device);
-    DeterminantFromPivotedLUKernel<<<
-        config.block_count, config.thread_per_block, 0, device.stream()>>>(
-        config.virtual_thread_count, n, lu_factor_ptr, pivots, output_ptr,
-        info);
-  }
-};
-
-template struct DeterminantFromPivotedLUFunctor;
-template struct DeterminantFromPivotedLUFunctor;
-template struct DeterminantFromPivotedLUFunctor;
-template struct DeterminantFromPivotedLUFunctor;
-
 template 
 __global__ void EyeKernel(Cuda3DLaunchConfig config, int batch_size, int m,
                           int n, Scalar* matrix_batch_ptr) {
   const int matrix_size = m * n;
-  const Scalar one = Const::make_const(1.0);
+  const Scalar one = Scalar(1);
   CUDA_AXIS_KERNEL_LOOP(batch, config.virtual_thread_count, x) {
     if (batch >= batch_size) {
       break;
@@ -205,16 +57,14 @@ template 
 struct EyeFunctor {
   void operator()(const GPUDevice& device,
                   typename TTypes::Tensor matrix_batch) {
-    using CudaType = typename CUDAComplexT::type;
     const int batch_size = matrix_batch.dimension(0);
     const int m = matrix_batch.dimension(1);
     const int n = matrix_batch.dimension(2);
-    CudaType* matrix_batch_ptr =
-        reinterpret_cast(matrix_batch.data());
     Cuda3DLaunchConfig config = GetCuda3DLaunchConfig(batch_size, m, n, device,
                                                       EyeKernel, 0, 0);
     EyeKernel<<>>(config, batch_size, m, n, matrix_batch_ptr);
+                device.stream()>>>(config, batch_size, m, n,
+                                   matrix_batch.data());
   }
 };
 
diff --git a/tensorflow/core/kernels/determinant_op.cc b/tensorflow/core/kernels/determinant_op.cc
index 876dbff030..b06f42384e 100644
--- a/tensorflow/core/kernels/determinant_op.cc
+++ b/tensorflow/core/kernels/determinant_op.cc
@@ -14,10 +14,13 @@ limitations under the License.
 ==============================================================================*/
 
 // See docs in ../ops/linalg_ops.cc.
+
 #include 
 
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/kernels/determinant_op.h"
 #endif
 
 #include "third_party/eigen3/Eigen/LU"
@@ -31,23 +34,24 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 
 #if GOOGLE_CUDA
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/kernels/cuda_solvers.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #endif
 
 namespace tensorflow {
 
-// A helper function to compute the sign and absolute value of the
-// log of the determinant of inputs via a partially pivoted LU
+// A helper function to compute the sign and absolute value of the log of the
+// determinant of inputs via a partially pivoted LU
 // factorization.
 //
-// Returns the sign in 'sign' and the log determinant in 'logdet'
+// Returns the log of the absolute value of the determinant, and its sign in
+// 'sign'.
 template 
-static void SLogDet(
+static typename Eigen::NumTraits::Real SLogDet(
     const Eigen::Matrix& inputs,
-    Scalar* sign, Scalar* log_abs_det) {
-  *log_abs_det = 0;
+    Scalar* sign) {
+  using RealScalar = typename Eigen::NumTraits::Real;
+  RealScalar log_abs_det = 0;
   *sign = 1;
   // An empty matrix' determinant is defined to be 1.
   // (https://en.wikipedia.org/wiki/Determinant)
@@ -58,27 +62,25 @@ static void SLogDet(
     Eigen::Matrix LU = lu.matrixLU();
     *sign = lu.permutationP().determinant();
     auto diag = LU.diagonal().array().eval();
-    auto abs_diag = diag.cwiseAbs().template cast().eval();
-    *log_abs_det += abs_diag.log().sum();
+    auto abs_diag = diag.cwiseAbs().eval();
+    log_abs_det += abs_diag.log().sum();
     *sign *= (diag / abs_diag).prod();
   }
-  if (!Eigen::numext::isfinite(*log_abs_det)) {
+  if (!Eigen::numext::isfinite(log_abs_det)) {
     *sign = 0;
-    *log_abs_det = std::log(0.0);
+    log_abs_det =
+        log_abs_det > 0 ? -std::log(RealScalar(0)) : std::log(RealScalar(0));
   }
+  return log_abs_det;
 }
 
 template 
 class LogDeterminantOp : public LinearAlgebraOp {
  public:
-  typedef LinearAlgebraOp Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit LogDeterminantOp(OpKernelConstruction* context) : Base(context) {}
 
-  using TensorShapes = typename Base::TensorShapes;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   TensorShapes GetOutputMatrixShapes(
       const TensorShapes& input_matrix_shapes) const final {
     return TensorShapes({TensorShape({}), TensorShape({})});
@@ -87,9 +89,9 @@ class LogDeterminantOp : public LinearAlgebraOp {
   void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs,
                      MatrixMaps* outputs) final {
     Scalar sign;
-    Scalar log_abs_det;
-    SLogDet(Eigen::Matrix(inputs[0]),
-            &sign, &log_abs_det);
+    const RealScalar log_abs_det = SLogDet(
+        Eigen::Matrix(inputs[0]),
+        &sign);
 
     outputs->at(0)(0, 0) = sign;
     outputs->at(1)(0, 0) = log_abs_det;
@@ -99,14 +101,10 @@ class LogDeterminantOp : public LinearAlgebraOp {
 template 
 class DeterminantOp : public LinearAlgebraOp {
  public:
-  typedef LinearAlgebraOp Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit DeterminantOp(OpKernelConstruction* context) : Base(context) {}
 
-  using TensorShapes = typename Base::TensorShapes;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   TensorShapes GetOutputMatrixShapes(
       const TensorShapes& input_matrix_shape) const final {
     return TensorShapes({TensorShape({})});
@@ -115,15 +113,10 @@ class DeterminantOp : public LinearAlgebraOp {
   void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs,
                      MatrixMaps* outputs) final {
     Scalar sign;
-    Scalar log_abs_det;
-    SLogDet(Eigen::Matrix(inputs[0]),
-            &sign, &log_abs_det);
-    Scalar determinant = sign * std::exp(log_abs_det);
-    // TODO(rmlarsen): Don't fail on infinite determinants, since that could
-    // be a valid result and the user should check for it instead.
-    OP_REQUIRES(context, Eigen::numext::isfinite(determinant),
-                errors::InvalidArgument("The determinant is not finite."));
-    outputs->at(0)(0, 0) = determinant;
+    const RealScalar log_abs_det = SLogDet(
+        Eigen::Matrix(inputs[0]),
+        &sign);
+    outputs->at(0)(0, 0) = sign * std::exp(log_abs_det);
   }
 };
 
@@ -171,7 +164,7 @@ class DeterminantOpGpu : public AsyncOpKernel {
       return;
     }
 
-    // TODO(rmlarsen): Convert to std::make_unique when available.
+    // TODO(rmlarsen): Convert to absl::make_unique when available.
     std::unique_ptr solver(new CudaSolver(context));
 
     // Reuse the input buffer or make a copy for the factorization step,
@@ -255,18 +248,160 @@ class DeterminantOpGpu : public AsyncOpKernel {
         for (int i = 0; i < host_infos[0].size(); ++i) {
           // It is OK for a matrix to be singular (signaled by info > 0),
           // corresponding to determinant of zero, but we do want to catch
-          // invalid arguments to GetrfBatched.
+          // invalid arguments to Getrf{Batched}.
           OP_REQUIRES_ASYNC(
-              context,
-              host_infos[0].data()[i] >= 0 ||
-                  host_infos[0].data()[i] == kint32min,
+              context, host_infos[0](i) >= 0,
               errors::InvalidArgument("Invalid input argument no. ",
                                       host_infos[0].data()[i],
                                       " for batch index ", i, "."),
               done);
+        }
+      }
+      done();
+    };
+    CudaSolver::CheckLapackInfoAndDeleteSolverAsync(std::move(solver), dev_info,
+                                                    std::move(info_checker));
+  }
+};
+
+template 
+class LogDeterminantOpGpu : public AsyncOpKernel {
+ public:
+  explicit LogDeterminantOpGpu(OpKernelConstruction* context)
+      : AsyncOpKernel(context) {}
+
+  void ComputeAsync(OpKernelContext* context, DoneCallback done) final {
+    const Tensor& input = context->input(0);
+    const int ndims = input.dims();
+    const int64 n = input.dim_size(ndims - 1);
+    // Validate inputs.
+    OP_REQUIRES_ASYNC(
+        context, ndims >= 2,
+        errors::InvalidArgument("Input must have rank >= 2, got ", ndims),
+        done);
+    OP_REQUIRES_ASYNC(
+        context, input.dim_size(ndims - 2) == n,
+        errors::InvalidArgument("Input matrices must be square, got",
+                                input.dim_size(ndims - 2), " != ", n),
+        done);
+
+    // Allocate output.
+    TensorShape out_shape;
+    for (int dim = 0; dim < ndims - 2; ++dim) {
+      out_shape.AddDim(input.dim_size(dim));
+    }
+    out_shape.AppendShape(TensorShape({}));
+    Tensor* sign;
+    OP_REQUIRES_OK_ASYNC(context, context->allocate_output(0, out_shape, &sign),
+                         done);
+    Tensor* log_abs_det;
+    OP_REQUIRES_OK_ASYNC(
+        context, context->allocate_output(1, out_shape, &log_abs_det), done);
+
+    // By definition, the determinant of an empty matrix is equal to one.
+    const GPUDevice& d = context->eigen_device();
+    if (input.NumElements() == 0) {
+      functor::SetOneFunctor one_func;
+      one_func(d, sign->template flat());
+      functor::SetZeroFunctor zero_func;
+      zero_func(d, log_abs_det->template flat());
+      done();
+      return;
+    }
+
+    // TODO(rmlarsen): Convert to absl::make_unique when available.
+    std::unique_ptr solver(new CudaSolver(context));
+
+    // Reuse the input buffer or make a copy for the factorization step,
+    // depending on whether this ops owns it exclusively.
+    Tensor input_copy;
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        solver->forward_input_or_allocate_scoped_tensor(
+            {0}, DataTypeToEnum::value, input.shape(), &input_copy),
+        done);
+    if (!input.SharesBufferWith(input_copy)) {
+      d.memcpy(input_copy.flat().data(), input.flat().data(),
+               input.NumElements() * sizeof(Scalar));
+    }
+    auto input_copy_reshaped = input_copy.template flat_inner_dims();
+    const int64 batch_size = input_copy_reshaped.dimension(0);
+
+    // Allocate pivots on the device.
+    Tensor pivots;
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        solver->allocate_scoped_tensor(DataTypeToEnum::value,
+                                       TensorShape{batch_size, n}, &pivots),
+        done);
+    auto pivots_mat = pivots.template matrix();
+
+    // Prepare pointer arrays for cuBlas' batch interface.
+    // TODO(rmlarsen): Find a way to encode pointer arrays in pinned host memory
+    // without the ugly casting.
+    auto input_copy_ptrs = solver->GetScratchSpace(
+        sizeof(Scalar*) * batch_size, "input_copy_ptrs",
+        /* on_host */ true);
+
+    // Compute the partially pivoted LU factorization(s) of the matrix/matrices.
+    std::vector dev_info;
+    if (n / batch_size <= 128) {
+      // For small matrices or large batch sizes, we use the batched interface
+      // from cuBlas.
+      const Scalar** input_copy_ptrs_base =
+          reinterpret_cast(input_copy_ptrs.mutable_data());
+      for (int batch = 0; batch < batch_size; ++batch) {
+        input_copy_ptrs_base[batch] = &input_copy_reshaped(batch, 0, 0);
+      }
+      dev_info.push_back(
+          solver->GetDeviceLapackInfo(batch_size, "getrfBatched"));
+      OP_REQUIRES_OK_ASYNC(
+          context,
+          solver->GetrfBatched(n, input_copy_ptrs_base, n, pivots_mat.data(),
+                               &dev_info.back(), batch_size),
+          done);
+    } else {
+      // For large matrices or small batch sizes we use the non-batched
+      // interface from cuSolver, which is much faster for large matrices.
+      dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrf"));
+      for (int batch = 0; batch < batch_size; ++batch) {
+        OP_REQUIRES_OK_ASYNC(
+            context,
+            solver->Getrf(n, n, &input_copy_reshaped(batch, 0, 0), n,
+                          &pivots_mat(batch, 0), &dev_info.back()(batch)),
+            done);
+      }
+    }
+
+    auto input_copy_reshaped_const =
+        const_cast(&input_copy)
+            ->template flat_inner_dims();
+    auto sign_reshaped = sign->flat();
+    auto log_abs_det_reshaped = log_abs_det->flat();
+    // Compute the determinant for each batch as (-1)^s * prod(diag(U)),
+    // where s is the order of the permutation encoded in pivots and U is the
+    // upper triangular factor of the LU factorization, which is written to
+    // input_copy by the Getrf{Batched} kernel.
+    functor::LogDeterminantFromPivotedLUFunctor functor;
+    functor(d, input_copy_reshaped_const, pivots_mat.data(), sign_reshaped,
+            log_abs_det_reshaped);
+
+    // Register callback to check info after kernels finish.
+    auto info_checker = [context, done](
+                            const Status& status,
+                            const std::vector& host_infos) {
+      if (!status.ok() && errors::IsInvalidArgument(status) &&
+          !host_infos.empty()) {
+        for (int i = 0; i < host_infos[0].size(); ++i) {
+          // It is OK for a matrix to be singular (signaled by info > 0),
+          // corresponding to determinant of zero, but we do want to catch
+          // invalid arguments to Getrf{Batched}.
           OP_REQUIRES_ASYNC(
-              context, host_infos[0].data()[i] != kint32min,
-              errors::InvalidArgument("The determinant is not finite."), done);
+              context, host_infos[0](i) >= 0,
+              errors::InvalidArgument("Invalid input argument no. ",
+                                      host_infos[0].data()[i],
+                                      " for batch index ", i, "."),
+              done);
         }
       }
       done();
@@ -282,6 +417,15 @@ REGISTER_LINALG_OP_GPU("MatrixDeterminant", (DeterminantOpGpu),
                        complex64);
 REGISTER_LINALG_OP_GPU("MatrixDeterminant", (DeterminantOpGpu),
                        complex128);
+
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu),
+                       float);
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu),
+                       double);
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu),
+                       complex64);
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant",
+                       (LogDeterminantOpGpu), complex128);
 #endif  // GOOGLE_CUDA
 
 REGISTER_LINALG_OP("MatrixDeterminant", (DeterminantOp), float);
diff --git a/tensorflow/core/kernels/determinant_op.h b/tensorflow/core/kernels/determinant_op.h
new file mode 100644
index 0000000000..e931e328e4
--- /dev/null
+++ b/tensorflow/core/kernels/determinant_op.h
@@ -0,0 +1,47 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
+
+#include "tensorflow/core/framework/tensor_types.h"
+
+namespace tensorflow {
+namespace functor {
+
+// Helper functor to compute Determinant from a partially pivoted LU
+// factorization.
+template 
+struct DeterminantFromPivotedLUFunctor {
+  void operator()(const Device& device,
+                  typename TTypes::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes::Tensor output,
+                  int* info);
+};
+
+// Helper functor to compute sign and log of the absolute value of the
+// determinant from a partially pivoted LU factorization.
+template 
+struct LogDeterminantFromPivotedLUFunctor {
+  void operator()(const Device& device,
+                  typename TTypes::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes::Tensor sign,
+                  typename TTypes::Tensor log_abs_det);
+};
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
diff --git a/tensorflow/core/kernels/determinant_op_gpu.cu.cc b/tensorflow/core/kernels/determinant_op_gpu.cu.cc
new file mode 100644
index 0000000000..c866204c97
--- /dev/null
+++ b/tensorflow/core/kernels/determinant_op_gpu.cu.cc
@@ -0,0 +1,168 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/determinant_op.h"
+
+#include 
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/cuda_solvers.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+
+namespace tensorflow {
+namespace functor {
+
+typedef Eigen::GpuDevice GPUDevice;
+namespace {
+__device__ int PermutationOrder(int n, const int* pivots) {
+  // Compute the order of the permutation from the number of transpositions
+  // encoded in the pivot array, see:
+  // http://icl.cs.utk.edu/lapack-forum/viewtopic.php?f=2&t=340
+  int order = 0;
+  for (int i = 0; i < n - 1; ++i) {
+    // Notice: Internally, the cuBlas code uses Fortran convention (1-based)
+    // indexing so we expect pivots[i] == i + 1 for rows that were not moved.
+    order += pivots[i] != (i + 1);
+  }
+  return order;
+}
+
+#if defined(__CUDACC__)
+// Hack around missing support for complex in NVCC.
+template 
+__device__ inline std::complex complex_multiply(const std::complex& a,
+                                                   const std::complex& b) {
+  const T a_real = Eigen::numext::real(a);
+  const T a_imag = Eigen::numext::imag(a);
+  const T b_real = Eigen::numext::real(b);
+  const T b_imag = Eigen::numext::imag(b);
+  return std::complex(a_real * b_real - a_imag * b_imag,
+                         a_real * b_imag + a_imag * b_real);
+}
+__device__ inline complex64 operator*(const complex64& a, const complex64& b) {
+  return complex_multiply(a, b);
+}
+__device__ inline complex64 operator*(const complex64& a, const float& b) {
+  return complex64(Eigen::numext::real(a) * b, Eigen::numext::imag(a) * b);
+}
+__device__ inline complex64 operator/(const complex64& a, const float& b) {
+  const float inv_b = 1.0f / b;
+  return a * inv_b;
+}
+__device__ inline complex128 operator*(const complex128& a,
+                                       const complex128& b) {
+  return complex_multiply(a, b);
+}
+__device__ inline complex128 operator*(const complex128& a, const double& b) {
+  return complex128(Eigen::numext::real(a) * b, Eigen::numext::imag(a) * b);
+}
+__device__ inline complex128 operator/(const complex128& a, const double& b) {
+  const double inv_b = 1.0 / b;
+  return a * inv_b;
+}
+#endif
+}  // namespace
+
+// This kernel computes either determinant or log_abs_determinant, depending
+// on the value of the template parameter. If compute_log_abs_det is false,
+// the sign argument is ignored.
+template 
+__global__ void DeterminantFromPivotedLUKernel(int nthreads, int n,
+                                               const Scalar* lu_factor,
+                                               const int* all_pivots,
+                                               Scalar* sign,
+                                               Scalar* log_abs_det) {
+  typedef typename Eigen::NumTraits::Real RealScalar;
+  const int matrix_size = n * n;
+  const int stride = n + 1;
+  // We only parallelize over batches here. Performance is not critical,
+  // since this cheap O(n) kernel always follows an O(n^3) LU factorization.
+  // The main purpose is to avoid having to copy the LU decomposition to
+  // host memory.
+  CUDA_1D_KERNEL_LOOP(o_idx, nthreads) {
+    // Initialize sign to (-1)^order.
+    const int order = PermutationOrder(n, all_pivots + o_idx * n);
+    Scalar prod_sign = order % 2 ? Scalar(-1) : Scalar(1);
+    RealScalar sum_log_abs_det = RealScalar(0);
+    int i_idx = matrix_size * o_idx;
+    for (int i = 0; i < n; ++i, i_idx += stride) {
+      const RealScalar abs_i = Eigen::numext::abs(lu_factor[i_idx]);
+      sum_log_abs_det += Eigen::numext::log(abs_i);
+      prod_sign = prod_sign * (lu_factor[i_idx] / abs_i);
+    }
+    if (!Eigen::numext::isfinite(sum_log_abs_det)) {
+      prod_sign = Scalar(0);
+      sum_log_abs_det = sum_log_abs_det > 0 ? -Eigen::numext::log(RealScalar(0))
+                                            : Eigen::numext::log(RealScalar(0));
+    }
+    if (compute_log_abs_det) {
+      sign[o_idx] = prod_sign;
+      log_abs_det[o_idx] = Scalar(sum_log_abs_det);
+    } else {
+      log_abs_det[o_idx] = prod_sign * Eigen::numext::exp(sum_log_abs_det);
+    }
+  }
+}
+
+template 
+struct DeterminantFromPivotedLUFunctor {
+  void operator()(const GPUDevice& device,
+                  typename TTypes::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes::Tensor output,
+                  int* info) {
+    const int64 num_matrices = output.size();
+    const int64 n = lu_factor.dimension(2);
+    CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device);
+    DeterminantFromPivotedLUKernel
+        <<>>(
+            config.virtual_thread_count, n, lu_factor.data(), pivots, nullptr,
+            output.data());
+  }
+};
+
+template struct DeterminantFromPivotedLUFunctor;
+template struct DeterminantFromPivotedLUFunctor;
+template struct DeterminantFromPivotedLUFunctor;
+template struct DeterminantFromPivotedLUFunctor;
+
+template 
+struct LogDeterminantFromPivotedLUFunctor {
+  void operator()(const GPUDevice& device,
+                  typename TTypes::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes::Tensor sign,
+                  typename TTypes::Tensor log_abs_det) {
+    const int64 num_matrices = sign.size();
+    const int64 n = lu_factor.dimension(2);
+    CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device);
+    DeterminantFromPivotedLUKernel
+        <<>>(
+            config.virtual_thread_count, n, lu_factor.data(), pivots,
+            sign.data(), log_abs_det.data());
+  }
+};
+
+template struct LogDeterminantFromPivotedLUFunctor;
+template struct LogDeterminantFromPivotedLUFunctor;
+template struct LogDeterminantFromPivotedLUFunctor;
+template struct LogDeterminantFromPivotedLUFunctor;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/linalg_ops_common.h b/tensorflow/core/kernels/linalg_ops_common.h
index 1d31786728..f7c3f1950b 100644
--- a/tensorflow/core/kernels/linalg_ops_common.h
+++ b/tensorflow/core/kernels/linalg_ops_common.h
@@ -172,13 +172,14 @@ extern template class LinearAlgebraOp;
 
 }  // namespace tensorflow
 
-#define INHERIT_LINALG_TYPEDEFS(Scalar)                   \
-  typedef LinearAlgebraOp Base;                   \
-  using Matrix = typename Base::Matrix;                   \
-  using MatrixMap = typename Base::MatrixMap;             \
-  using MatrixMaps = typename Base::MatrixMaps;           \
-  using ConstMatrixMap = typename Base::ConstMatrixMap;   \
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps; \
+#define INHERIT_LINALG_TYPEDEFS(Scalar)                       \
+  typedef LinearAlgebraOp Base;                       \
+  using RealScalar = typename Eigen::NumTraits::Real; \
+  using Matrix = typename Base::Matrix;                       \
+  using MatrixMap = typename Base::MatrixMap;                 \
+  using MatrixMaps = typename Base::MatrixMaps;               \
+  using ConstMatrixMap = typename Base::ConstMatrixMap;       \
+  using ConstMatrixMaps = typename Base::ConstMatrixMaps;     \
   using TensorShapes = typename Base::TensorShapes;
 
 #define REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) \
diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc
index 64edfe470d..cae84f52d7 100644
--- a/tensorflow/core/kernels/matrix_inverse_op.cc
+++ b/tensorflow/core/kernels/matrix_inverse_op.cc
@@ -69,7 +69,6 @@ class MatrixInverseOp : public LinearAlgebraOp {
     // a result of basic user mistakes, such as providing integer valued
     // matrices that are exactly singular, or due to underflow if this
     // code is run with denormals being flushed to zero.
-    using RealScalar = typename Base::RealScalar;
     const RealScalar min_abs_pivot =
         lu_decomposition.matrixLU().diagonal().cwiseAbs().minCoeff();
     OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc
index 2e4098dfab..169f3dae76 100644
--- a/tensorflow/core/kernels/matrix_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_solve_op.cc
@@ -44,18 +44,12 @@ static const char kErrMsg[] = "Input matrix is not invertible.";
 template 
 class MatrixSolveOp : public LinearAlgebraOp {
  public:
-  typedef LinearAlgebraOp Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit MatrixSolveOp(OpKernelConstruction* context) : Base(context) {
     OP_REQUIRES_OK(context, context->GetAttr("adjoint", &adjoint_));
   }
 
-  using TensorShapes = typename Base::TensorShapes;
-  using Matrix = typename Base::Matrix;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMap = typename Base::ConstMatrixMap;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   void ValidateInputMatrixShapes(
       OpKernelContext* context,
       const TensorShapes& input_matrix_shapes) const final {
@@ -102,7 +96,6 @@ class MatrixSolveOp : public LinearAlgebraOp {
     // a result of basic user mistakes such providing integer valued
     // matrices that are exactly singular, or due to underflow if this
     // code is run with denormals being flushed to zero.
-    using RealScalar = typename Base::RealScalar;
     const RealScalar min_abs_pivot =
         lu_decomposition.matrixLU().diagonal().cwiseAbs().minCoeff();
     OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op.cc b/tensorflow/core/kernels/matrix_triangular_solve_op.cc
index 953f37fa02..6f7e6a7496 100644
--- a/tensorflow/core/kernels/matrix_triangular_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_triangular_solve_op.cc
@@ -47,7 +47,7 @@ perftools::gputools::DeviceMemory AsDeviceMemory(
 template 
 class MatrixTriangularSolveOp : public LinearAlgebraOp {
  public:
-  typedef LinearAlgebraOp Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit MatrixTriangularSolveOp(OpKernelConstruction* context)
       : Base(context), lower_(true), adjoint_(false) {
@@ -55,13 +55,6 @@ class MatrixTriangularSolveOp : public LinearAlgebraOp {
     OP_REQUIRES_OK(context, context->GetAttr("adjoint", &adjoint_));
   }
 
-  using TensorShapes = typename Base::TensorShapes;
-  using Matrix = typename Base::Matrix;
-  using MatrixMap = typename Base::MatrixMap;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMap = typename Base::ConstMatrixMap;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   void ValidateInputMatrixShapes(
       OpKernelContext* context,
       const TensorShapes& input_matrix_shapes) const final {
@@ -97,7 +90,6 @@ class MatrixTriangularSolveOp : public LinearAlgebraOp {
       // an empty set of equation as the empty matrix.
       return;
     }
-    using RealScalar = typename Base::RealScalar;
     const RealScalar min_abs_pivot = matrix.diagonal().cwiseAbs().minCoeff();
     OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
                 errors::InvalidArgument("Input matrix is not invertible."));
diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py
index 7368fbc4a1..222038b22e 100644
--- a/tensorflow/python/kernel_tests/determinant_op_test.py
+++ b/tensorflow/python/kernel_tests/determinant_op_test.py
@@ -126,11 +126,10 @@ class DeterminantOpTest(test.TestCase):
     self._compareDeterminant(
         np.random.rand(3, 4, 5, 2, 2).astype(np.complex128))
 
-  def testOverflow(self):
+  def testInfiniteDeterminant(self):
     max_double = np.finfo("d").max
     huge_matrix = np.array([[max_double, 0.0], [0.0, max_double]])
-    with self.assertRaisesOpError("not finite"):
-      self._compareDeterminant(huge_matrix)
+    self._compareDeterminant(huge_matrix)
 
   def testNonSquareMatrix(self):
     # When the determinant of a non-square matrix is attempted we should return
-- 
GitLab


From 720efa37a4e93d5833e6e928993790f2523f0d85 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Tue, 24 Oct 2017 10:41:58 -0700
Subject: [PATCH 309/573] Roll forward CL 171084886

171084886 had to be rolled back twice due to various open source build issues.
I'm trying again, now that I think I've addressed all the pertinent issues.

Original CL description:

Don't use dlsym to resolve symbols in the CPU JIT

Instead of resolving symbols via dlsym when JITting for the CPU backend, use a
registry based mechanism.  This lets us kill off the --export_dynamic hack that
we used to need for CustomCall on the CPU backend.

PiperOrigin-RevId: 173277862
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   3 +-
 .../index_ops_kernel_argmax_float_1d.cc       |   3 +
 .../index_ops_kernel_argmax_float_2d.cc       |   3 +
 tensorflow/compiler/xla/service/cpu/BUILD     |  12 ++
 .../cpu/custom_call_target_registry.cc        |  39 ++++
 .../service/cpu/custom_call_target_registry.h |  74 +++++++
 .../xla/service/cpu/simple_orc_jit.cc         | 198 ++++++++++--------
 tensorflow/compiler/xla/tests/BUILD           |   3 +-
 .../compiler/xla/tests/custom_call_test.cc    |  14 +-
 tensorflow/compiler/xla/xla.bzl               |   8 -
 10 files changed, 259 insertions(+), 98 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 4ee7989824..2b43e313eb 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -5,7 +5,6 @@ package(
 )
 
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 
 tf_kernel_library(
     name = "xla_ops",
@@ -153,6 +152,7 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_1d.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
@@ -164,6 +164,7 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_2d.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
index afbd64ca50..47cf8c6675 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -47,3 +48,5 @@ EIGEN_STRONG_INLINE void argmax_float_1d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_1d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_1d_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(argmax_float_1d_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
index 841ff2f4df..9b83392d8f 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -49,3 +50,5 @@ EIGEN_STRONG_INLINE void argmax_float_2d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_2d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_2d_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(argmax_float_2d_xla_impl);
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 136cbe7cb7..56bc1a6706 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -153,6 +153,7 @@ cc_library(
         ":cpu_runtime_avx",
         ":cpu_runtime_neon",
         ":cpu_runtime_sse4_1",
+        ":custom_call_target_registry",
         ":disassembler",
         ":external_constant_pool",
         ":runtime_conv2d",
@@ -719,6 +720,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "custom_call_target_registry",
+    srcs = [
+        "custom_call_target_registry.cc",
+    ],
+    hdrs = [
+        "custom_call_target_registry.h",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
new file mode 100644
index 0000000000..5f5803874b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
@@ -0,0 +1,39 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
+
+namespace xla {
+namespace cpu {
+
+CustomCallTargetRegistry* CustomCallTargetRegistry::Global() {
+  static auto* registry = new CustomCallTargetRegistry;
+  return registry;
+}
+
+void CustomCallTargetRegistry::Register(const std::string& symbol,
+                                        void* address) {
+  std::lock_guard lock(mu_);
+  registered_symbols_[symbol] = address;
+}
+
+void* CustomCallTargetRegistry::Lookup(const std::string& symbol) const {
+  std::lock_guard lock(mu_);
+  auto it = registered_symbols_.find(symbol);
+  return it == registered_symbols_.end() ? nullptr : it->second;
+}
+
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
new file mode 100644
index 0000000000..2994642356
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
@@ -0,0 +1,74 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
+
+// This file is depended on by kernels that have to build for mobile devices.
+// For this reason, we avoid relying on TensorFlow and instead only use the
+// standard C++ library.
+
+#include   // NOLINT
+#include 
+#include 
+
+namespace xla {
+namespace cpu {
+
+// The CPU JIT compiler uses this registry to resolve symbolic CustomCall
+// targets; so when using the CPU JIT, CustomCall targets need to be registered
+// here with the symbol name used in the CustomCall.
+//
+// The XLA AOT compiler links using a standard offline linker; so when compiling
+// in AOT mode, you *also* need to make sure the name of the callee (presumably
+// implemented in C++) matches up with the symbolic name used in the CustomCall.
+//
+// We maintain the registry in both the JIT and the AOT cases for simplicity,
+// but we only use it when running in JIT mode.
+class CustomCallTargetRegistry {
+ public:
+  static CustomCallTargetRegistry* Global();
+
+  void Register(const std::string& symbol, void* address);
+  void* Lookup(const std::string& symbol) const;
+
+ private:
+  std::unordered_map registered_symbols_;
+  mutable std::mutex mu_;
+};
+
+class RegisterCustomCallTarget {
+ public:
+  explicit RegisterCustomCallTarget(const std::string& name, void* address) {
+    CustomCallTargetRegistry::Global()->Register(name, address);
+  }
+};
+
+#define REGISTER_CUSTOM_CALL_CONCAT(a, b) a##b
+
+#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, counter) \
+  static ::xla::cpu::RegisterCustomCallTarget REGISTER_CUSTOM_CALL_CONCAT(    \
+      custom_call_target_register, counter)(symbol,                           \
+                                            reinterpret_cast(address))
+
+#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(symbol, address) \
+  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, __COUNTER__)
+
+#define REGISTER_CUSTOM_CALL_TARGET(function) \
+  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(#function, function)
+
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index cfffb3fbc3..fdf02e5b42 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_avx.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
@@ -43,81 +44,6 @@ namespace xla {
 namespace cpu {
 namespace {
 
-// Converts a symbol 'name' into the form expected by dlsym().
-std::string CanonicalizeSymbol(const std::string& name) {
-#if defined(__APPLE__)
-  // On Mac OS X, dlsym() expects names not to be prefixed with a leading
-  // underscore.
-  if (!name.empty() && name.front() == '_') {
-    return name.substr(1);
-  }
-#endif
-  return name;
-}
-
-class JITSymbolTable {
- public:
-  JITSymbolTable() { Populate(); }
-
-  void* Lookup(llvm::StringRef jit_symbol_name) const {
-    auto it = jit_symbol_table_.find(jit_symbol_name);
-    return it == jit_symbol_table_.end() ? nullptr : it->getValue();
-  }
-
-  static bool MustBeInTable(llvm::StringRef name) {
-    // In particular, names starting with
-    // runtime::kXlaCpuRuntimeSymbolNamePrefix should not be dlsym'ed.
-    return name.startswith(runtime::kXlaCpuRuntimeSymbolNamePrefix);
-  }
-
- private:
-  void AddJITSymbolToTable(llvm::StringRef jit_symbol_name,
-                           llvm::StringRef cpp_symbol_name,
-                           void* jit_symbol_value) {
-    // The JIT symbol name and the C++ symbol name (with an extern "C" linkage)
-    // need to match, otherwise AOT links will fail.
-    CHECK(jit_symbol_name == cpp_symbol_name);
-    CHECK(jit_symbol_table_.insert({jit_symbol_name, jit_symbol_value}).second);
-  }
-
-  void Populate() {
-#define ADD_JIT_SYMBOL_TO_TABLE(base_name)                       \
-  do {                                                           \
-    AddJITSymbolToTable(                                         \
-        xla::cpu::runtime::k##base_name##SymbolName,             \
-        "__xla_cpu_runtime_" #base_name,                         \
-        reinterpret_cast(__xla_cpu_runtime_##base_name)); \
-  } while (false)
-
-    ADD_JIT_SYMBOL_TO_TABLE(AcquireInfeedBufferForDequeue);
-    ADD_JIT_SYMBOL_TO_TABLE(ReleaseInfeedBufferAfterDequeue);
-    ADD_JIT_SYMBOL_TO_TABLE(AcquireOutfeedBufferForPopulation);
-    ADD_JIT_SYMBOL_TO_TABLE(ReleaseOutfeedBufferAfterPopulation);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV8F32AVX);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV8F32AVX);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32SSE);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32SSE);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32NEON);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32NEON);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenConvF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF64);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
-    ADD_JIT_SYMBOL_TO_TABLE(ParallelForkJoin);
-
-#undef ADD_JIT_SYMBOL_TO_TABLE
-  }
-
-  llvm::StringMap jit_symbol_table_;
-};
-
-const JITSymbolTable& GetJITSymbolTable() {
-  static JITSymbolTable* symbol_table = new JITSymbolTable;
-  return *symbol_table;
-}
-
 // A simple SymbolResolver that delegates to the host dynamic linker.
 class SimpleResolver : public llvm::JITSymbolResolver {
  public:
@@ -125,7 +51,6 @@ class SimpleResolver : public llvm::JITSymbolResolver {
       : external_constant_pool_(external_constant_pool) {}
 
   llvm::JITSymbol findSymbol(const std::string& name) override {
-    string name_as_string(name);
     if (const uint8* from_constant_pool =
             external_constant_pool_->Find(string(name))) {
       return llvm::JITEvaluatedSymbol(
@@ -133,13 +58,7 @@ class SimpleResolver : public llvm::JITSymbolResolver {
           llvm::JITSymbolFlags::None);
     }
 
-    std::string canonical_name = CanonicalizeSymbol(name);
-    const JITSymbolTable& jit_symbol_table = GetJITSymbolTable();
-
-    void* func_addr = JITSymbolTable::MustBeInTable(canonical_name)
-                          ? jit_symbol_table.Lookup(canonical_name)
-                          : dlsym(RTLD_DEFAULT, canonical_name.c_str());
-
+    void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name);
     if (func_addr == nullptr) {
       return nullptr;
     }
@@ -255,5 +174,118 @@ llvm::JITSymbol SimpleOrcJIT::FindSymbol(const std::string& name) {
   return nullptr;
 }
 
+namespace {
+// Register some known symbols with the CustomCallTargetRegistry.
+bool RegisterKnownJITSymbols() {
+  CustomCallTargetRegistry* registry = CustomCallTargetRegistry::Global();
+
+#define REGISTER_CPU_RUNTIME_SYMBOL(base_name)                                \
+  do {                                                                        \
+    auto* function_address =                                                  \
+        reinterpret_cast(__xla_cpu_runtime_##base_name);               \
+    registry->Register(xla::cpu::runtime::k##base_name##SymbolName,           \
+                       function_address);                                     \
+    CHECK_EQ(                                                                 \
+        tensorflow::StringPiece(xla::cpu::runtime::k##base_name##SymbolName), \
+        "__xla_cpu_runtime_" #base_name);                                     \
+  } while (false)
+
+  REGISTER_CPU_RUNTIME_SYMBOL(AcquireInfeedBufferForDequeue);
+  REGISTER_CPU_RUNTIME_SYMBOL(AcquireOutfeedBufferForPopulation);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenConvF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32NEON);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32SSE);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV8F32AVX);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32NEON);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32SSE);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV8F32AVX);
+  REGISTER_CPU_RUNTIME_SYMBOL(ParallelForkJoin);
+  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue);
+  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation);
+
+#undef REGISTER_CPU_RUNTIME_SYMBOL
+
+#define REGISTER_LIBM_SYMBOL(name)                                    \
+  do {                                                                \
+    /* Register both the F32 and F64 variants of the libm symbol.  */ \
+    registry->Register(#name "f", reinterpret_cast(name##f));  \
+    registry->Register(#name, reinterpret_cast(name));         \
+  } while (false)
+
+  REGISTER_LIBM_SYMBOL(acos);
+  REGISTER_LIBM_SYMBOL(acosh);
+  REGISTER_LIBM_SYMBOL(asin);
+  REGISTER_LIBM_SYMBOL(asinh);
+  REGISTER_LIBM_SYMBOL(atan);
+  REGISTER_LIBM_SYMBOL(atan2);
+  REGISTER_LIBM_SYMBOL(atanh);
+  REGISTER_LIBM_SYMBOL(cbrt);
+  REGISTER_LIBM_SYMBOL(ceil);
+  REGISTER_LIBM_SYMBOL(copysign);
+  REGISTER_LIBM_SYMBOL(cos);
+  REGISTER_LIBM_SYMBOL(cosh);
+  REGISTER_LIBM_SYMBOL(erf);
+  REGISTER_LIBM_SYMBOL(erfc);
+  REGISTER_LIBM_SYMBOL(exp);
+  REGISTER_LIBM_SYMBOL(exp2);
+  REGISTER_LIBM_SYMBOL(expm1);
+  REGISTER_LIBM_SYMBOL(fabs);
+  REGISTER_LIBM_SYMBOL(fdim);
+  REGISTER_LIBM_SYMBOL(floor);
+  REGISTER_LIBM_SYMBOL(fma);
+  REGISTER_LIBM_SYMBOL(fmax);
+  REGISTER_LIBM_SYMBOL(fmin);
+  REGISTER_LIBM_SYMBOL(fmod);
+  REGISTER_LIBM_SYMBOL(frexp);
+  REGISTER_LIBM_SYMBOL(hypot);
+  REGISTER_LIBM_SYMBOL(ilogb);
+  REGISTER_LIBM_SYMBOL(ldexp);
+  REGISTER_LIBM_SYMBOL(lgamma);
+  REGISTER_LIBM_SYMBOL(llrint);
+  REGISTER_LIBM_SYMBOL(llround);
+  REGISTER_LIBM_SYMBOL(log);
+  REGISTER_LIBM_SYMBOL(log10);
+  REGISTER_LIBM_SYMBOL(log1p);
+  REGISTER_LIBM_SYMBOL(log2);
+  REGISTER_LIBM_SYMBOL(logb);
+  REGISTER_LIBM_SYMBOL(lrint);
+  REGISTER_LIBM_SYMBOL(lround);
+  REGISTER_LIBM_SYMBOL(modf);
+  REGISTER_LIBM_SYMBOL(nan);
+  REGISTER_LIBM_SYMBOL(nearbyint);
+  REGISTER_LIBM_SYMBOL(nextafter);
+  REGISTER_LIBM_SYMBOL(nexttoward);
+  REGISTER_LIBM_SYMBOL(pow);
+  REGISTER_LIBM_SYMBOL(remainder);
+  REGISTER_LIBM_SYMBOL(remquo);
+  REGISTER_LIBM_SYMBOL(rint);
+  REGISTER_LIBM_SYMBOL(round);
+  REGISTER_LIBM_SYMBOL(scalbln);
+  REGISTER_LIBM_SYMBOL(scalbn);
+  REGISTER_LIBM_SYMBOL(sin);
+  REGISTER_LIBM_SYMBOL(sincos);
+  REGISTER_LIBM_SYMBOL(sinh);
+  REGISTER_LIBM_SYMBOL(sqrt);
+  REGISTER_LIBM_SYMBOL(tan);
+  REGISTER_LIBM_SYMBOL(tanh);
+  REGISTER_LIBM_SYMBOL(tgamma);
+  REGISTER_LIBM_SYMBOL(trunc);
+
+#undef REGISTER_LIBM_SYMBOL
+
+  registry->Register("memcpy", reinterpret_cast(memcpy));
+  registry->Register("memmove", reinterpret_cast(memmove));
+  registry->Register("memset", reinterpret_cast(memset));
+  return true;
+}
+
+bool unused = RegisterKnownJITSymbols();
+}  // namespace
+
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 43127925e6..2ea7b9bd8e 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -23,7 +23,6 @@ filegroup(
     ]),
 )
 
-load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test_library")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "generate_backend_suites")
@@ -988,13 +987,13 @@ xla_test(
 xla_test(
     name = "custom_call_test",
     srcs = ["custom_call_test.cc"],
-    linkopts = export_dynamic_linkopts,
     deps = [
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/custom_call_test.cc b/tensorflow/compiler/xla/tests/custom_call_test.cc
index 342478bc74..74f73a1ddc 100644
--- a/tensorflow/compiler/xla/tests/custom_call_test.cc
+++ b/tensorflow/compiler/xla/tests/custom_call_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -31,19 +32,19 @@ limitations under the License.
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/test.h"
 
-
-extern "C" void TF_EXPORT R0F32Add2(float* out, float** in) {
+namespace {
+void R0F32Add2(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float*));
   *out = **in + 2.0f;
 }
 
-extern "C" void TF_EXPORT R2F32ReduceSum(float* out, float** in) {
+void R2F32ReduceSum(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   *out = array[0] + array[1] + array[2] + array[3];
 }
 
-extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
+void Add1ToValues(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   out[0] = array[0] + 1;
@@ -51,6 +52,11 @@ extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
   out[2] = array[2] + 1;
   out[3] = array[3] + 1;
 }
+}  // namespace
+
+REGISTER_CUSTOM_CALL_TARGET(R0F32Add2);
+REGISTER_CUSTOM_CALL_TARGET(R2F32ReduceSum);
+REGISTER_CUSTOM_CALL_TARGET(Add1ToValues);
 
 namespace xla {
 namespace {
diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl
index 22e70ec97a..3fa5bcc1df 100644
--- a/tensorflow/compiler/xla/xla.bzl
+++ b/tensorflow/compiler/xla/xla.bzl
@@ -17,11 +17,3 @@ def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0):
                    protoc="@protobuf_archive//:protoc",
                    testonly=testonly,
                    visibility=visibility,)
-
-# Flags required for modules that export symbols that are to be called by the
-# XLA CustomCall operator. CustomCall must be able to find symbols with dlsym(),
-# which on Linux requires we link with --export-dynamic.
-export_dynamic_linkopts = select({
-    "//tensorflow:darwin": [],
-    "//conditions:default": ["-Wl,--export-dynamic"],
-})
-- 
GitLab


From 34f5d001c4224c8c1f4ce615bcb1e76610e95673 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 13:06:29 -0700
Subject: [PATCH 310/573] Fix K-FAC's loss_functions.insert_slice_in_zeros and
 add tests

PiperOrigin-RevId: 173299853
---
 .../contrib/kfac/python/kernel_tests/BUILD    | 12 +++++
 .../kernel_tests/loss_functions_test.py       | 44 +++++++++++++++++++
 .../contrib/kfac/python/ops/loss_functions.py | 11 ++---
 3 files changed, 62 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index 1b2a5cdd38..fd4f588741 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -79,6 +79,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "loss_functions_test",
+    srcs = ["loss_functions_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/kfac/python/ops:loss_functions",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+    ],
+)
+
 py_test(
     name = "optimizer_test",
     srcs = ["optimizer_test.py"],
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
new file mode 100644
index 0000000000..86dd839896
--- /dev/null
+++ b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
@@ -0,0 +1,44 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tf.contrib.kfac.loss_functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.kfac.python.ops import loss_functions
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+class InsertSliceInZerosTest(test.TestCase):
+
+  def testBadShape(self):
+    bad_shaped_ones = array_ops.ones(shape=[1, 3])  # n.b. shape[1] != 1
+    with self.assertRaises(ValueError):
+      loss_functions.insert_slice_in_zeros(bad_shaped_ones, 1, 42, 17)
+
+  def test3d(self):
+    input_tensor = constant_op.constant([[[1, 2]], [[3, 4]]])
+    expected_output_array = [[[1, 2], [0, 0]], [[3, 4], [0, 0]]]
+    op = loss_functions.insert_slice_in_zeros(input_tensor, 1, 2, 0)
+    with self.test_session() as sess:
+      actual_output_array = sess.run(op)
+    self.assertAllEqual(expected_output_array, actual_output_array)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py
index d80382b9cf..0b5c3d4928 100644
--- a/tensorflow/contrib/kfac/python/ops/loss_functions.py
+++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py
@@ -632,11 +632,12 @@ class MultiBernoulliNegativeLogProbLoss(DistributionNegativeLogProbLoss,
 
 
 def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position):
-  """Inserts slice into a larger tensors of zeros.
+  """Inserts slice into a larger tensor of zeros.
 
-  Forms a new tensor that which is the same shape as slice_, except that
+  Forms a new tensor which is the same shape as slice_to_insert, except that
   the dimension given by 'dim' is expanded to the size given by 'dim_size'.
-  'position' determines the position (index) of the slice in that dimension.
+  'position' determines the position (index) at which to insert the slice within
+  that dimension.
 
   Assumes slice_to_insert.shape[dim] = 1.
 
@@ -644,7 +645,7 @@ def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position):
     slice_to_insert: The slice to insert.
     dim: The dimension which to expand with zeros.
     dim_size: The new size of the 'dim' dimension.
-    position: The position of 'slice_' in the new tensor.
+    position: The position of 'slice_to_insert' in the new tensor.
 
   Returns:
     The new tensor.
@@ -662,4 +663,4 @@ def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position):
   before[dim] = position
   after[dim] = dim_size - position - 1
 
-  return array_ops.pad(slice_to_insert, zip(before, after))
+  return array_ops.pad(slice_to_insert, list(zip(before, after)))
-- 
GitLab


From 6d6c8e012bb4498bbdd75de3f64f0cab72b1391c Mon Sep 17 00:00:00 2001
From: Jeremy Sharpe 
Date: Tue, 24 Oct 2017 16:16:09 -0400
Subject: [PATCH 311/573] Fix a typo of "Jenkins".

---
 tensorflow/tools/ci_build/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md
index acef833909..202fcb9101 100644
--- a/tensorflow/tools/ci_build/README.md
+++ b/tensorflow/tools/ci_build/README.md
@@ -50,7 +50,7 @@ and tests. Click on **Details** to see the results from Jenkins or the internal
 CI system.
 
 Results from Jenkins are displayed in the Jenkins UI. For more information,
-see the [Jenkns documentation](https://jenkins.io/doc/).
+see the [Jenkins documentation](https://jenkins.io/doc/).
 
 Results from the internal CI system are displayed in the Build Status UI. In
 this UI, to see the logs for a failed build:
-- 
GitLab


From 03f1105003c7e30127ed9449524c36d2c384b79c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 13:30:04 -0700
Subject: [PATCH 312/573] Better support for Metrics in graph mode: * Avoid
 situation where variables are created in the wrong graph. * Add an
 init_variables() method that returns an op that will initialize   any
 non-initialized variables.

PiperOrigin-RevId: 173302832
---
 tensorflow/contrib/eager/python/BUILD         |  3 ++
 .../contrib/eager/python/metrics_impl.py      | 32 ++++++++++++++++++-
 .../contrib/eager/python/metrics_test.py      | 11 +++++--
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 3d7d307778..ee2ec79141 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -131,10 +131,13 @@ py_library(
     deps = [
         "//tensorflow/contrib/summary:summary_ops",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:layers_base",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:context",
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 959ee735b0..77a84e006e 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -23,16 +23,30 @@ import re
 from tensorflow.contrib.summary import summary_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 
 
 _to_replace = re.compile("[^A-Za-z0-9.]")
 
 
+def _init_var(v):
+  def do_init(v):
+    with ops.control_dependencies([v.assign(v.initial_value)]):
+      return constant_op.constant(True)
+  return control_flow_ops.cond(
+      resource_variable_ops.var_is_initialized_op(v._handle),  # pylint: disable=protected-access
+      lambda: constant_op.constant(False),
+      lambda: do_init(v))
+
+
 class Metric(object):
   """A metric holds state for aggregating statistics over an evaluation run.
 
@@ -76,7 +90,10 @@ class Metric(object):
     if context.in_graph_mode():
       # We make self.call() into a graph callable here, so that we can
       # return a single op that performs all of the variable updates.
+      self._construction_scope = ops.get_default_graph().as_default
       self.call = function.defun(self.call)
+    else:
+      self._construction_scope = context.eager_mode
 
   # ---- API for users ----
   def __call__(self, *args, **kwargs):
@@ -89,7 +106,8 @@ class Metric(object):
       **kwargs: A mini-batch of inputs to the Metric, passed on to `call()`.
     """
     if not self._built:
-      with variable_scope.variable_scope(self._scope):
+      with variable_scope.variable_scope(
+          self._scope), self._construction_scope():
         self.build(*args, **kwargs)
       self._built = True
     return self.call(*args, **kwargs)
@@ -102,6 +120,18 @@ class Metric(object):
   def variables(self):
     return self._vars
 
+  def init_variables(self):
+    """Return an op for initializing this Metric's uninitialized variables.
+
+    Only for graph execution. Should be called after variables are created
+    in the first execution of __call__().
+
+    Returns:
+      An op to run.
+    """
+    assert context.in_graph_mode()
+    return control_flow_ops.group(*[_init_var(v) for v in self._vars])
+
   # ---- To be implemented by descendants ---
   def build(self, *args, **kwargs):
     """Method to create variables.
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 1880e762d4..fce6be1761 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -29,7 +29,6 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.lib.io import tf_record
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.training import training_util
 
@@ -136,11 +135,16 @@ class MetricsTest(test.TestCase):
       m = metrics.Mean()
       p = array_ops.placeholder(dtypes.float32)
       accumulate = m(p)
-      variables.global_variables_initializer().run()
+      init_op = m.init_variables()
+      init_op.run()
       sess.run(accumulate, feed_dict={p: [1, 10, 100]})
       sess.run(accumulate, feed_dict={p: 1000})
       sess.run(accumulate, feed_dict={p: [10000, 100000]})
       self.assertAllEqual(m.result().eval(), 111111.0/6)
+      # Second init is ignored, since the variables are already initialized.
+      init_op.run()
+      sess.run(accumulate, feed_dict={p: 7})
+      self.assertAllEqual(m.result().eval(), 111118.0/7)
 
   def testTwoMeansGraph(self):
     # Verify two metrics with the same class and name don't
@@ -150,7 +154,8 @@ class MetricsTest(test.TestCase):
       m2 = metrics.Mean()
       accumulate1 = m1(0)
       accumulate2 = m2(2)
-      variables.global_variables_initializer().run()
+      m1.init_variables().run()
+      m2.init_variables().run()
       sess.run([accumulate1, accumulate2])
       self.assertEqual(0, m1.result().eval())
       self.assertEqual(2, m2.result().eval())
-- 
GitLab


From 73f8b044ea7333b25ef5c9841c1e072e45ad5890 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 13:48:27 -0700
Subject: [PATCH 313/573] replace min for std:min to avoid issues with clang
 compilation e.g.:
 http://ci.tensorflow.org/job/nightly-matrix-linux-gpu-clang/159/console

PiperOrigin-RevId: 173305545
---
 tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc  |  4 ++--
 tensorflow/core/kernels/reduction_gpu_kernels.cu.h | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
index d82676ff7e..6d3758fef1 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
@@ -209,7 +209,7 @@ void LSTMBlockCellFpropWithCUDA(
   // Use 2D blocks. The number of threads per block is equal to x * y, where x =
   // min(batch_size, 8) and y = 32. See above for guidance on number of
   // threads.
-  dim3 block_dim_2d(min(batch_size, 8), 32);
+  dim3 block_dim_2d(std::min(batch_size, 8), 32);
   dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast(block_dim_2d.x)),
                    Eigen::divup(cell_size, static_cast(block_dim_2d.y)));
 
@@ -323,7 +323,7 @@ void LSTMBlockCellBpropWithCUDA(
     const bool use_peephole) {
   const cudaStream_t& cu_stream = GetCudaStream(ctx);
 
-  dim3 block_dim_2d(min(batch_size, 8), 32);
+  dim3 block_dim_2d(std::min(batch_size, 8), 32);
   dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast(block_dim_2d.x)),
                    Eigen::divup(cell_size, static_cast(block_dim_2d.y)));
 
diff --git a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
index be9a611881..36ca7f834f 100644
--- a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
+++ b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
@@ -460,7 +460,7 @@ void LaunchScalarReduction(OpKernelContext* ctx, OUT_T out, IN_T in,
     return;
   } else if (in_size <= 1 << 19) {
     const int num_threads = 256;
-    const int num_blocks = min(32, Eigen::divup(in_size, num_threads));
+    const int num_blocks = std::min(32, Eigen::divup(in_size, num_threads));
     // it seems like tailoring this to the GPU
     // would be more effective, but all attempts
     // at making this a multiple of the number of
@@ -557,13 +557,13 @@ void LaunchColumnReduction_LTE16Cols(OpKernelContext* ctx, OUT_T out, IN_T in,
                                      int extent_x, int extent_y, Op op, T init,
                                      const cudaStream_t& cu_stream) {
   int rows_per_warp = 32 / extent_y;
-  dim3 block_dim(32, min(Eigen::divup(extent_x, rows_per_warp), 32), 1);
+  dim3 block_dim(32, std::min(Eigen::divup(extent_x, rows_per_warp), 32), 1);
   dim3 grid_dim(1,
                 Eigen::divup(static_cast(extent_x),
                              rows_per_warp * block_dim.y),
                 1);
 
-  grid_dim.y = min((int)grid_dim.y, 32);
+  grid_dim.y = std::min((int)grid_dim.y, 32);
 
   if (grid_dim.y > 2 && grid_dim.y < 32) {
     int log2 = Log2Floor(grid_dim.y);
@@ -596,10 +596,10 @@ template 
 void LaunchColumnReduction_LTE4096Cols(OpKernelContext* ctx, OUT_T out, IN_T in,
                                        int extent_x, int extent_y, Op op,
                                        T init, const cudaStream_t& cu_stream) {
-  dim3 block_dim(32, min(extent_x, 32), 1);
+  dim3 block_dim(32, std::min(extent_x, 32), 1);
   dim3 grid_dim((extent_y + 31) / 32, 1, 1);
 
-  if (grid_dim.x < 16) grid_dim.y = min((extent_x + 31) / 32, 32);
+  if (grid_dim.x < 16) grid_dim.y = std::min((extent_x + 31) / 32, 32);
 
   if (grid_dim.y > 2 && grid_dim.y < 32) {
     int log2 = Log2Floor(grid_dim.y);
-- 
GitLab


From bf1fad214febef6af5c101d8f953d0109c46dfbb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 13:55:04 -0700
Subject: [PATCH 314/573] Fix NCCL rewrite bug when rerunning sessions
 (assigned device id is not stable). Fix collocate_gradients for initial
 losses. Remove NcclBroadcast gradient test for now. The generated AddN to
 accumulate the broadcast outputs before passing it to the gradient function
 is CPU only and cannot be collocated with NcclBroadcast on the GPU.

PiperOrigin-RevId: 173306409
---
 .../contrib/nccl/kernels/nccl_rewrite.cc      |  9 +-
 .../contrib/nccl/python/ops/nccl_ops_test.py  | 10 +--
 tensorflow/python/ops/gradients_impl.py       | 87 +++++++++----------
 3 files changed, 54 insertions(+), 52 deletions(-)

diff --git a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
index 94a77c59da..a4de46a93f 100644
--- a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
+++ b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
@@ -117,6 +117,7 @@ Status ReplaceBroadcast(Graph* graph, Node* node) {
   TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype));
   int send_dev = node->assigned_device_name_index();
   int num_devices = 0;  // Number of distinct devices, incremented below.
+  std::vector recv_index_map;  // Map device name index to stable index.
 
   // Map device name index to nodes that take the broadcast as input.
   std::vector> out_nodes_map;
@@ -126,9 +127,11 @@ Status ReplaceBroadcast(Graph* graph, Node* node) {
                       : edge->dst()->assigned_device_name_index();
     if (out_nodes_map.size() <= dst_dev) {
       out_nodes_map.resize(dst_dev + 1);
+      recv_index_map.resize(dst_dev + 1);
     }
     auto it = out_nodes_map.begin() + dst_dev;
     if (it->empty()) {
+      recv_index_map[dst_dev] = num_devices;
       ++num_devices;
     }
     it->emplace_front(NodeBuilder::NodeOut(edge->dst(), edge->dst_input()));
@@ -211,16 +214,18 @@ Status ReplaceBroadcast(Graph* graph, Node* node) {
     if (out_nodes_map[recv_dev].empty()) {
       continue;
     }
+    int recv_index = recv_index_map[recv_dev];
     if (is_fully_defined) {
       // If the shape is fully defined, define one const node per device.
-      NodeBuilder shape_builder(strings::StrCat(shape_name, recv_dev), "Const");
+      NodeBuilder shape_builder(strings::StrCat(shape_name, recv_index),
+                                "Const");
       shape_builder.Attr("value", tensor_proto).Attr("dtype", DT_INT32);
       TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node));
       shape_node->set_assigned_device_name_index(recv_dev);
     }
     Node* recv_node;
     TF_RETURN_IF_ERROR(
-        make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_dev))
+        make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_index))
             .Input(shape_node)
             .Finalize(graph, &recv_node));
     recv_node->set_assigned_device_name_index(recv_dev);
diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
index 255409303a..0b13e3595e 100644
--- a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
+++ b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
@@ -117,7 +117,8 @@ class NcclTestCase(test.TestCase):
       inputs = [array_ops.placeholder(t.dtype, t.shape) for t in tensors]
       reduce_tensors = nccl_reduce(inputs, devices)
       losses = _DeviceTensors(tensors, [t.device for t in reduce_tensors])
-      grads = gradients.gradients(reduce_tensors, inputs, losses)
+      grads = gradients.gradients(
+          reduce_tensors, inputs, losses, colocate_gradients_with_ops=True)
       return [g for g in grads if g is not None]
 
     self._Test(_Gradient, numpy_fn)
@@ -159,7 +160,7 @@ class BroadcastTest(NcclTestCase):
   def testBroadcastSingleDevice(self):
     # Broadcasts on a single device are removed completely during rewrite.
     self._Test(_NcclBroadcast, lambda x, y: x,
-               (['/device:GPU:0', '/device:GPU:0']))
+               (['/device:GPU:0', '/device:GPU:0'],))
 
   def testBroadcastToCpuError(self):
     # Broadcasts to CPU is not supported.
@@ -167,10 +168,7 @@ class BroadcastTest(NcclTestCase):
         errors.NotFoundError,
         "No registered '_NcclBroadcastRecv' OpKernel for CPU devices"):
       self._Test(_NcclBroadcast, lambda x, y: x,
-                 (['/device:GPU:0', '/device:CPU:0']))
-
-  def testBroadcastGrad(self):
-    self._TestGradient(_NcclBroadcast, lambda x, y: x + y)
+                 (['/device:GPU:0', '/device:CPU:0'],))
 
 
 class CombinedTest(NcclTestCase):
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index f7b72eb82f..eb34a35a2b 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -227,53 +227,52 @@ def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops):
   for i in xrange(len(grad_ys)):
     grad_y = grad_ys[i]
     y = ys[i]
-    if grad_y is None:
-      if y.dtype.is_complex:
-        raise TypeError(
-            "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
-            y.dtype)
-      with _maybe_colocate_with(y.op, colocate_gradients_with_ops):
+    with _maybe_colocate_with(y.op, colocate_gradients_with_ops):
+      if grad_y is None:
+        if y.dtype.is_complex:
+          raise TypeError(
+              "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
+              y.dtype)
         new_grad_ys.append(array_ops.fill(
             array_ops.shape(y), constant_op.constant(
                 1, dtype=y.dtype, name="grad_ys_%d" % i)))
-      continue
-    if y.dtype.is_floating or y.dtype.is_integer:
-      if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
-        raise TypeError("Gradient type %s generated for real or "
-                        "integer-valued tensor %s with type %s must be "
-                        "real or integer" %
-                        (dtypes.as_dtype(grad_y.dtype).name, y,
-                         dtypes.as_dtype(y.dtype).name))
-    elif y.dtype.is_complex:
-      if not grad_y.dtype.is_complex:
-        raise TypeError("Gradient type %s generated for complex-valued "
-                        "tensor %s with type %s must be real" %
-                        (dtypes.as_dtype(grad_y.dtype).name, y,
-                         dtypes.as_dtype(y.dtype).name))
-    else:
-      raise TypeError("Tensor %s with type %s must be numeric "
-                      "to obtain a default gradient" %
-                      (y, dtypes.as_dtype(y.dtype).name))
-    # Create a grad_y tensor in the name scope of the gradient.
-    # Required for TensorArrays to identify which gradient call a
-    # grad_y value is coming from.
-    if isinstance(grad_y, ops.IndexedSlices):
-      new_grad_ys.append(
-          ops.IndexedSlices(
-              indices=(array_ops.identity(grad_y.indices,
-                                          name="grad_ys_%d_indices" % i)
-                       if isinstance(grad_y.indices, ops.Tensor)
-                       else grad_y.indices),
-              values=(array_ops.identity(grad_y.values,
-                                         name="grad_ys_%d_values" % i)
-                      if isinstance(grad_y.values, ops.Tensor)
-                      else grad_y.values),
-              dense_shape=(array_ops.identity(grad_y.dense_shape,
-                                              name="grad_ys_%d_shape" % i)
-                           if isinstance(grad_y.dense_shape, ops.Tensor)
-                           else grad_y.dense_shape)))
-    else:
-      new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
+        continue
+      if y.dtype.is_floating or y.dtype.is_integer:
+        if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
+          raise TypeError("Gradient type %s generated for real or "
+                          "integer-valued tensor %s with type %s must be "
+                          "real or integer" %
+                          (dtypes.as_dtype(grad_y.dtype).name, y,
+                           dtypes.as_dtype(y.dtype).name))
+      elif y.dtype.is_complex:
+        if not grad_y.dtype.is_complex:
+          raise TypeError("Gradient type %s generated for complex-valued "
+                          "tensor %s with type %s must be real" %
+                          (dtypes.as_dtype(grad_y.dtype).name, y,
+                           dtypes.as_dtype(y.dtype).name))
+      else:
+        raise TypeError("Tensor %s with type %s must be numeric "
+                        "to obtain a default gradient" %
+                        (y, dtypes.as_dtype(y.dtype).name))
+      # Create a grad_y tensor in the name scope of the gradient.
+      # Required for TensorArrays to identify which gradient call a
+      # grad_y value is coming from.
+      if isinstance(grad_y, ops.IndexedSlices):
+        new_grad_ys.append(
+            ops.IndexedSlices(
+                indices=(array_ops.identity(
+                    grad_y.indices, name="grad_ys_%d_indices" % i)
+                         if isinstance(grad_y.indices, ops.Tensor) else
+                         grad_y.indices),
+                values=(array_ops.identity(
+                    grad_y.values, name="grad_ys_%d_values" % i) if isinstance(
+                        grad_y.values, ops.Tensor) else grad_y.values),
+                dense_shape=(array_ops.identity(
+                    grad_y.dense_shape, name="grad_ys_%d_shape" % i)
+                             if isinstance(grad_y.dense_shape, ops.Tensor) else
+                             grad_y.dense_shape)))
+      else:
+        new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
 
   return new_grad_ys
 
-- 
GitLab


From 134daeb4151349acf8c2b3c22f5aebc3e429d756 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Oct 2017 13:55:22 -0700
Subject: [PATCH 315/573] Eager reuse story is False instead of AUTO_REUSE.

We want variables with eager execution to have object semantics instead of name semantics and this is a small step in that direction.

This means that the functional style layer invocations (tf.layers.dense() etc.)
will NOT work when eager execution is enabled. Instead, use of the object-oriented
layers is advised.

PiperOrigin-RevId: 173306447
---
 .../kernel_tests/variable_scope_test.py       | 62 +++++++++----------
 tensorflow/python/ops/variable_scope.py       | 62 +++++++++----------
 2 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 0ea58b4402..29f583d5ba 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -39,7 +39,6 @@ from tensorflow.python.platform import test
 
 class VariableScopeTest(test.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes()
   def testGetVar(self):
     vs = variable_scope._get_default_variable_store()
     v = vs.get_variable("v", [1])
@@ -52,7 +51,6 @@ class VariableScopeTest(test.TestCase):
     v1 = vs.get_variable("v", [1], use_resource=True)
     self.assertTrue(isinstance(v1, resource_variable_ops.ResourceVariable))
 
-  @test_util.run_in_graph_and_eager_modes()
   def testNameExists(self):
     vs = variable_scope._get_default_variable_store()
     # No check by default, so we can both create and get existing names.
@@ -60,15 +58,14 @@ class VariableScopeTest(test.TestCase):
     v1 = vs.get_variable("v", [1])
     self.assertEqual(v, v1)
 
-    if context.in_graph_mode():
-      # When reuse is False, we fail when variables are already there.
-      vs.get_variable("w", [1], reuse=False)  # That's ok.
-      with self.assertRaises(ValueError):
-        vs.get_variable("v", [1], reuse=False)  # That fails.
-      # When reuse is True, we fail when variables are new.
-      vs.get_variable("v", [1], reuse=True)  # That's ok.
-      with self.assertRaises(ValueError):
-        vs.get_variable("u", [1], reuse=True)  # That fails.
+    # When reuse is False, we fail when variables are already there.
+    vs.get_variable("w", [1], reuse=False)  # That's ok.
+    with self.assertRaises(ValueError):
+      vs.get_variable("v", [1], reuse=False)  # That fails.
+    # When reuse is True, we fail when variables are new.
+    vs.get_variable("v", [1], reuse=True)  # That's ok.
+    with self.assertRaises(ValueError):
+      vs.get_variable("u", [1], reuse=True)  # That fails.
 
   @test_util.run_in_graph_and_eager_modes()
   def testNamelessStore(self):
@@ -224,10 +221,12 @@ class VariableScopeTest(test.TestCase):
         self.assertAllClose(self.evaluate(losses[1]), 0.4)
         self.assertAllClose(self.evaluate(losses[2]), 0.5)
       with variable_scope.variable_scope("foo", reuse=True):
-        v = variable_scope.get_variable("v",
-                                        [])  # "v" is alredy there, reused
-        losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)
-        self.assertEqual(3, len(losses))  # No new loss added.
+        # reuse=True is for now only supported when eager execution is disabled.
+        if context.in_graph_mode():
+          v = variable_scope.get_variable("v",
+                                          [])  # "v" is alredy there, reused
+          losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)
+          self.assertEqual(3, len(losses))  # No new loss added.
 
   @test_util.run_in_graph_and_eager_modes()
   def testInitializeFromValue(self):
@@ -439,20 +438,20 @@ class VariableScopeTest(test.TestCase):
       with variable_scope.variable_scope(vs, reuse=False) as jump_no_reuse:
         self.assertFalse(jump_no_reuse.reuse)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testVarScopeGetOrCreateReuse(self):
-    def test_value(value):
-      x = constant_op.constant(value)
-      with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
-                                         reuse=variable_scope.AUTO_REUSE):
-        _ = state_ops.assign(variable_scope.get_variable("var", []), x)
-      with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
-                                         reuse=variable_scope.AUTO_REUSE):
-        _ = variable_scope.get_variable("var", [])
-      self.assertEqual(value, self.evaluate(x))
-    test_value(42.)  # Variable is created.
-    test_value(13.)  # Variable is reused hereafter.
-    test_value(17.)
+    with self.test_session():
+      def test_value(value):
+        x = constant_op.constant(value)
+        with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
+                                           reuse=variable_scope.AUTO_REUSE):
+          _ = state_ops.assign(variable_scope.get_variable("var", []), x)
+        with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
+                                           reuse=variable_scope.AUTO_REUSE):
+          _ = variable_scope.get_variable("var", [])
+        self.assertEqual(value, x.eval())
+      test_value(42.)  # Variable is created.
+      test_value(13.)  # Variable is reused hereafter.
+      test_value(17.)
 
   def testVarOpScope(self):
     with self.test_session():
@@ -745,9 +744,10 @@ class VariableScopeTest(test.TestCase):
                        ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
 
     # Check that local variable respects `reuse`.
-    with variable_scope.variable_scope(outer, "default", reuse=True):
-      self.assertEqual(
-          variable_scope.get_local_variable("w", []).name, "outer/w:0")
+    if context.in_graph_mode():
+      with variable_scope.variable_scope(outer, "default", reuse=True):
+        self.assertEqual(
+            variable_scope.get_local_variable("w", []).name, "outer/w:0")
 
   def testGetVarWithDevice(self):
     g = ops.Graph()
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 22048a0cef..8c5c639b68 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -259,8 +259,8 @@ class _VariableStore(object):
         applying it on a newly created variable will be added to the collection
         GraphKeys.REGULARIZATION_LOSSES and can be used for regularization.
       reuse: a Boolean, None, or tf.AUTO_REUSE. Controls reuse or creation
-        of variables. In Eager mode, this argument is always forced to be
-        tf.AUTO_REUSE.
+        of variables. When eager execution is enabled  this argument is always
+        forced to be False.
       trainable: If `True` also add the variable to the graph collection
         `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
       collections: List of graph collections keys to add the `Variable` to.
@@ -279,7 +279,8 @@ class _VariableStore(object):
       use_resource: If False, creates a regular Variable. If True, creates
         instead an experimental ResourceVariable which has well-defined
         semantics. Defaults to False (will later change to True).
-        In Eager mode, this argument is always forced to be true.
+        When eager execution is enabled this argument is always forced to be
+        true.
       custom_getter: Callable that takes as a first argument the true getter,
         and allows overwriting the internal get_variable method.
         The signature of `custom_getter` should match that of this method,
@@ -314,7 +315,7 @@ class _VariableStore(object):
           "Passed a custom_getter which is not callable: %s" % custom_getter)
 
     if context.in_eager_mode():
-      reuse = AUTO_REUSE
+      reuse = False
       use_resource = True
 
     # If a *_ref type is passed in an error would be triggered further down the
@@ -506,7 +507,7 @@ class _VariableStore(object):
     """
     if context.in_eager_mode():
       raise NotImplementedError("Partitioned variables are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
 
     initializing_from_value = initializer is not None and isinstance(
         initializer, ops.Tensor)
@@ -710,15 +711,6 @@ class _VariableStore(object):
     Raises:
       ValueError: See documentation of get_variable above.
     """
-    # Fast-path for get_variable in eager mode when the variable already
-    # exists. Note this skips error validation code, so mismatched shapes and
-    # dtypes will be caught when the variable is used instead of when the call
-    # to get_variable happens.
-    if context.in_eager_mode():
-      v = self._vars.get(name, None)
-      if v is not None:
-        return v
-
     # Set to true if initializer is a constant.
     initializing_from_value = False
     if initializer is not None and not callable(initializer):
@@ -732,6 +724,9 @@ class _VariableStore(object):
     if name in self._vars:
       # Here we handle the case when returning an existing variable.
       if reuse is False:
+        if context.in_eager_mode():
+          raise ValueError(
+              "Trying to recreate existing variable: %s" % self._vars[name])
         tb = self._vars[name].op.traceback[::-1]
         # Throw away internal tf entries and only take a few lines.
         tb = [x for x in tb if "tensorflow/python" not in x[0]][:3]
@@ -875,8 +870,8 @@ class VariableScope(object):
     initializer: default initializer passed to get_variable.
     regularizer: default regularizer passed to get_variable.
     reuse: Boolean, None, or tf.AUTO_REUSE, setting the reuse in
-      get_variable. In Eager mode, this argument is always forced to be
-      tf.AUTO_REUSE.
+      get_variable. When eager execution is enabled this argument is always
+      forced to be False.
     caching_device: string, callable, or None: the caching device passed to
       get_variable.
     partitioner: callable or `None`: the partitioner passed to `get_variable`.
@@ -885,8 +880,8 @@ class VariableScope(object):
     dtype: default type passed to get_variable (defaults to DT_FLOAT).
     use_resource: if False, create a normal Variable; if True create an
       experimental ResourceVariable with well-defined semantics. Defaults
-      to False (will later change to True). In Eager mode, this argument is
-      always forced to be True.
+      to False (will later change to True). When eager execution is enabled
+      this argument is always forced to be True.
     constraint: An optional projection function to be applied to the variable
       after being updated by an `Optimizer` (e.g. used to implement norm
       constraints or value constraints for layer weights). The function must
@@ -923,10 +918,10 @@ class VariableScope(object):
     if context.in_eager_mode():
       if self._caching_device is not None:
         raise NotImplementedError("Caching devices is not yet supported "
-                                  "in Eager mode.")
+                                  "when eager execution is enabled.")
       if self._partitioner is not None:
         raise NotImplementedError("Partitioned variables are not yet supported "
-                                  "in Eager mode.")
+                                  "when eager execution is enabled.")
       self._reuse = AUTO_REUSE
       self._use_resource = True
 
@@ -989,7 +984,8 @@ class VariableScope(object):
   def set_use_resource(self, use_resource):
     """Sets whether to use ResourceVariables for this scope."""
     if context.in_eager_mode() and not use_resource:
-      raise ValueError("In eager mode, use_resource cannot be set to false.")
+      raise ValueError("When eager execution is enabled, "
+                       "use_resource cannot be set to false.")
     self._use_resource = use_resource
 
   def set_regularizer(self, regularizer):
@@ -1000,14 +996,14 @@ class VariableScope(object):
     """Set caching_device for this scope."""
     if context.in_eager_mode():
       raise NotImplementedError("Caching devices are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
     self._caching_device = caching_device
 
   def set_partitioner(self, partitioner):
     """Set partitioner for this scope."""
     if partitioner and context.in_eager_mode():
       raise NotImplementedError("Partitioned variables are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
     self._partitioner = partitioner
 
   def set_custom_getter(self, custom_getter):
@@ -1062,7 +1058,7 @@ class VariableScope(object):
       if use_resource is None:
         use_resource = self._use_resource
     else:
-      reuse = AUTO_REUSE
+      reuse = False
       use_resource = True
 
     full_name = self.name + "/" + name if self.name else name
@@ -1108,7 +1104,7 @@ class VariableScope(object):
     """Gets an existing variable with this name or create a new one."""
     if context.in_eager_mode():
       raise NotImplementedError("Partitioned variables are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
     if initializer is None:
       initializer = self._initializer
     if regularizer is None:
@@ -1259,8 +1255,8 @@ Args:
       must be known.
   use_resource: If False, creates a regular Variable. If true, creates an
     experimental ResourceVariable instead with well-defined semantics.
-    Defaults to False (will later change to True). In Eager mode, this argument
-    is always forced to be True.
+    Defaults to False (will later change to True). When eager execution is
+    enabled this argument is always forced to be True.
   custom_getter: Callable that takes as a first argument the true getter, and
     allows overwriting the internal get_variable method.
     The signature of `custom_getter` should match that of this method,
@@ -1721,14 +1717,14 @@ class variable_scope(object):  # pylint: disable=invalid-name
       reuse: `True`, None, or tf.AUTO_REUSE; if `True`, we go into reuse mode
         for this scope as well as all sub-scopes; if tf.AUTO_REUSE, we create
         variables if they do not exist, and return them otherwise; if None, we
-        inherit the parent scope's reuse flag. In Eager mode, this argument is
-        always forced to be tf.AUTO_REUSE.
+        inherit the parent scope's reuse flag. When eager execution is enabled,
+        this argument is always forced to be tf.AUTO_REUSE.
       dtype: type of variables created in this scope (defaults to the type
         in the passed scope, or inherited from parent scope).
       use_resource: If False, all variables will be regular Variables. If True,
         experimental ResourceVariables with well-defined semantics will be used
-        instead. Defaults to False (will later change to True). In Eager mode,
-        this argument is always forced to be True.
+        instead. Defaults to False (will later change to True). When eager
+        execution is enabled this argument is always forced to be True.
       constraint: An optional projection function to be applied to the variable
         after being updated by an `Optimizer` (e.g. used to implement norm
         constraints or value constraints for layer weights). The function must
@@ -1935,8 +1931,8 @@ def variable(initial_value=None,
         caching_device=caching_device, name=name, dtype=dtype)
   elif not use_resource and context.in_eager_mode():
     raise RuntimeError(
-        "VariableScope should use resource variable in Eager mode, but "
-        "use_resource is False."
+        "VariableScope should use resource variable when eager execution is"
+        " enabled, but use_resource is False."
     )
   else:
     return variables.Variable(
-- 
GitLab


From de1b4a8a75ae3a50f4fa7480efb1177d79abf553 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 14:00:10 -0700
Subject: [PATCH 316/573] Refactor K-FAC FisherEstimator

PiperOrigin-RevId: 173307212
---
 .../contrib/kfac/python/kernel_tests/BUILD    |   2 +
 .../python/kernel_tests/estimator_test.py     |  68 +++++++---
 .../contrib/kfac/python/ops/estimator.py      | 116 +++++++++---------
 3 files changed, 115 insertions(+), 71 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index fd4f588741..8980f03092 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -13,6 +13,8 @@ py_test(
     deps = [
         "//tensorflow/contrib/kfac/python/ops:fisher_estimator",
         "//tensorflow/contrib/kfac/python/ops:layer_collection",
+        "//tensorflow/contrib/kfac/python/ops:utils",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
index 281274d884..b52a7b52a7 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
@@ -20,42 +20,80 @@ from __future__ import print_function
 
 from tensorflow.contrib.kfac.python.ops import estimator
 from tensorflow.contrib.kfac.python.ops import layer_collection as lc
+from tensorflow.contrib.kfac.python.ops import utils
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
+_ALL_ESTIMATION_MODES = ["gradients", "empirical", "curvature_prop", "exact"]
+
 
 class EstimatorTest(test.TestCase):
 
-  def testEstimatorInitManualRegistration(self):
-    with ops.Graph().as_default():
-      layer_collection = lc.LayerCollection()
+  def setUp(self):
+    self._graph = ops.Graph()
+    with self._graph.as_default():
+      self.layer_collection = lc.LayerCollection()
 
-      inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32)
-      weights = variable_scope.get_variable(
-          'w', shape=(2, 2), dtype=dtypes.float32)
-      bias = variable_scope.get_variable(
-          'b', initializer=init_ops.zeros_initializer(), shape=(2, 1))
-      output = math_ops.matmul(inputs, weights) + bias
+      self.inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32)
+      self.weights = variable_scope.get_variable(
+          "w", shape=(2, 2), dtype=dtypes.float32)
+      self.bias = variable_scope.get_variable(
+          "b", initializer=init_ops.zeros_initializer(), shape=(2, 1))
+      self.output = math_ops.matmul(self.inputs, self.weights) + self.bias
 
       # Only register the weights.
-      layer_collection.register_fully_connected((weights,), inputs, output)
+      self.layer_collection.register_fully_connected(
+          params=(self.weights,), inputs=self.inputs, outputs=self.output)
 
-      outputs = math_ops.tanh(output)
-      layer_collection.register_categorical_predictive_distribution(outputs)
+      self.outputs = math_ops.tanh(self.output)
+      self.targets = array_ops.zeros_like(self.outputs)
+      self.layer_collection.register_categorical_predictive_distribution(
+          logits=self.outputs, targets=self.targets)
 
+  def testEstimatorInitManualRegistration(self):
+    with self._graph.as_default():
       # We should be able to build an estimator for only the registered vars.
-      estimator.FisherEstimator([weights], 0.1, 0.2, layer_collection)
+      estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection)
 
       # Check that we throw an error if we try to build an estimator for vars
       # that were not manually registered.
       with self.assertRaises(ValueError):
-        estimator.FisherEstimator([weights, bias], 0.1, 0.2, layer_collection)
+        estimator.FisherEstimator([self.weights, self.bias], 0.1, 0.2,
+                                  self.layer_collection)
+
+      # Check that we throw an error if we don't include registered variables,
+      # i.e. self.weights
+      with self.assertRaises(ValueError):
+        estimator.FisherEstimator([], 0.1, 0.2, self.layer_collection)
+
+  @test.mock.patch.object(utils.SubGraph, "variable_uses", return_value=42)
+  def testVariableWrongNumberOfUses(self, mock_uses):
+    with self.assertRaises(ValueError):
+      estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection)
+
+  def testInvalidEstimationMode(self):
+    with self.assertRaises(ValueError):
+      estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection,
+                                "not_a_real_mode")
+
+  def testModeListCorrect(self):
+    with self._graph.as_default():
+      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
+                                      self.layer_collection)
+    self.assertItemsEqual(_ALL_ESTIMATION_MODES, est._gradient_fns.keys())
+
+  def testAllModesBuild(self):
+    for mode in _ALL_ESTIMATION_MODES:
+      with self._graph.as_default():
+        estimator.FisherEstimator([self.weights], 0.1, 0.2,
+                                  self.layer_collection, mode)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/kfac/python/ops/estimator.py b/tensorflow/contrib/kfac/python/ops/estimator.py
index c81086416c..6e2c9ecdce 100644
--- a/tensorflow/contrib/kfac/python/ops/estimator.py
+++ b/tensorflow/contrib/kfac/python/ops/estimator.py
@@ -80,6 +80,12 @@ class FisherEstimator(object):
     self._layers = layer_collection
     self._layers.create_subgraph()
     self._check_registration(variables)
+    self._gradient_fns = {
+        "gradients": self._get_grads_lists_gradients,
+        "empirical": self._get_grads_lists_empirical,
+        "curvature_prop": self._get_grads_lists_curvature_prop,
+        "exact": self._get_grads_lists_exact
+    }
     setup = self._setup(cov_ema_decay)
     self.cov_update_op, self.inv_update_op, self.inv_updates_dict = setup
 
@@ -201,75 +207,73 @@ class FisherEstimator(object):
     Raises:
       ValueError: If estimation_mode was improperly specified at construction.
     """
-    damping = self.damping
-
     fisher_blocks_list = self._layers.get_blocks()
-
     tensors_to_compute_grads = [
         fb.tensors_to_compute_grads() for fb in fisher_blocks_list
     ]
-    tensors_to_compute_grads_flat = nest.flatten(tensors_to_compute_grads)
-
-    if self._estimation_mode == "gradients":
-      grads_flat = gradients_impl.gradients(self._layers.total_sampled_loss(),
-                                            tensors_to_compute_grads_flat)
-      grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat)
-      grads_lists = tuple((grad,) for grad in grads_all)
-
-    elif self._estimation_mode == "empirical":
-      grads_flat = gradients_impl.gradients(self._layers.total_loss(),
-                                            tensors_to_compute_grads_flat)
-      grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat)
-      grads_lists = tuple((grad,) for grad in grads_all)
-
-    elif self._estimation_mode == "curvature_prop":
-      loss_inputs = list(loss.inputs for loss in self._layers.losses)
-      loss_inputs_flat = nest.flatten(loss_inputs)
-
-      transformed_random_signs = list(loss.multiply_fisher_factor(
-          utils.generate_random_signs(loss.fisher_factor_inner_shape))
-                                      for loss in self._layers.losses)
-
-      transformed_random_signs_flat = nest.flatten(transformed_random_signs)
-
-      grads_flat = gradients_impl.gradients(loss_inputs_flat,
-                                            tensors_to_compute_grads_flat,
-                                            grad_ys
-                                            =transformed_random_signs_flat)
-      grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat)
-      grads_lists = tuple((grad,) for grad in grads_all)
-
-    elif self._estimation_mode == "exact":
-      # Loop over all coordinates of all losses.
-      grads_all = []
-      for loss in self._layers.losses:
-        for index in np.ndindex(*loss.fisher_factor_inner_static_shape[1:]):
-          transformed_one_hot = loss.multiply_fisher_factor_replicated_one_hot(
-              index)
-          grads_flat = gradients_impl.gradients(loss.inputs,
-                                                tensors_to_compute_grads_flat,
-                                                grad_ys=transformed_one_hot)
-          grads_all.append(nest.pack_sequence_as(tensors_to_compute_grads,
-                                                 grads_flat))
-
-      grads_lists = zip(*grads_all)
-
-    else:
+
+    try:
+      grads_lists = self._gradient_fns[self._estimation_mode](
+          tensors_to_compute_grads)
+    except KeyError:
       raise ValueError("Unrecognized value {} for estimation_mode.".format(
           self._estimation_mode))
 
     for grads_list, fb in zip(grads_lists, fisher_blocks_list):
-      fb.instantiate_factors(grads_list, damping)
+      fb.instantiate_factors(grads_list, self.damping)
 
     cov_updates = [
         factor.make_covariance_update_op(cov_ema_decay)
         for factor in self._layers.get_factors()
     ]
-    inv_updates = {
-        op.name: op
-        for factor in self._layers.get_factors()
-        for op in factor.make_inverse_update_ops()
-    }
+    inv_updates = {op.name: op for op in self._get_all_inverse_update_ops()}
 
     return control_flow_ops.group(*cov_updates), control_flow_ops.group(
         *inv_updates.values()), inv_updates
+
+  def _get_all_inverse_update_ops(self):
+    for factor in self._layers.get_factors():
+      for op in factor.make_inverse_update_ops():
+        yield op
+
+  def _get_grads_lists_gradients(self, tensors):
+    grads_flat = gradients_impl.gradients(self._layers.total_sampled_loss(),
+                                          nest.flatten(tensors))
+    grads_all = nest.pack_sequence_as(tensors, grads_flat)
+    return tuple((grad,) for grad in grads_all)
+
+  def _get_grads_lists_empirical(self, tensors):
+    grads_flat = gradients_impl.gradients(self._layers.total_loss(),
+                                          nest.flatten(tensors))
+    grads_all = nest.pack_sequence_as(tensors, grads_flat)
+    return tuple((grad,) for grad in grads_all)
+
+  def _get_transformed_random_signs(self):
+    transformed_random_signs = []
+    for loss in self._layers.losses:
+      transformed_random_signs.append(
+          loss.multiply_fisher_factor(
+              utils.generate_random_signs(loss.fisher_factor_inner_shape)))
+    return transformed_random_signs
+
+  def _get_grads_lists_curvature_prop(self, tensors):
+    loss_inputs = list(loss.inputs for loss in self._layers.losses)
+    transformed_random_signs = self._get_transformed_random_signs()
+    grads_flat = gradients_impl.gradients(
+        nest.flatten(loss_inputs),
+        nest.flatten(tensors),
+        grad_ys=nest.flatten(transformed_random_signs))
+    grads_all = nest.pack_sequence_as(tensors, grads_flat)
+    return tuple((grad,) for grad in grads_all)
+
+  def _get_grads_lists_exact(self, tensors):
+    # Loop over all coordinates of all losses.
+    grads_all = []
+    for loss in self._layers.losses:
+      for index in np.ndindex(*loss.fisher_factor_inner_static_shape[1:]):
+        transformed_one_hot = loss.multiply_fisher_factor_replicated_one_hot(
+            index)
+        grads_flat = gradients_impl.gradients(
+            loss.inputs, nest.flatten(tensors), grad_ys=transformed_one_hot)
+        grads_all.append(nest.pack_sequence_as(tensors, grads_flat))
+    return zip(*grads_all)
-- 
GitLab


From 01365dbc2c257ff2ab409a2a5122a06739272737 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 14:10:14 -0700
Subject: [PATCH 317/573] Allow lists to be passed to tf.group().

PiperOrigin-RevId: 173308794
---
 tensorflow/python/ops/control_flow_ops.py      |  7 +++++--
 tensorflow/python/ops/control_flow_ops_test.py | 14 +++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index dcdbeefb70..10d8e01304 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -2910,7 +2910,7 @@ def _GroupControlDeps(dev, deps, name=None):
 def group(*inputs, **kwargs):
   """Create an op that groups multiple operations.
 
-  When this op finishes, all ops in `input` have finished. This op has no
+  When this op finishes, all ops in `inputs` have finished. This op has no
   output.
 
   See also @{tf.tuple$tuple} and
@@ -2938,7 +2938,10 @@ def group(*inputs, **kwargs):
 
     # Sorts *inputs according to their devices.
     ops_on_device = {}  # device -> operations specified on the device.
-    for inp in inputs:
+    for inp in nest.flatten(inputs):
+      if not hasattr(inp, "device"):
+        raise TypeError("Expected tf.group() expected Tensor arguments not "
+                        "'%s' with type '%s'" % (inp, type(inp)))
       if not hasattr(inp, "device"):
         if isinstance(inp, list):
           raise TypeError("To call tf.group() with a list, use "
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index 34c405f293..3e8f39dd24 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -115,11 +115,19 @@ class GroupTestCase(test_util.TensorFlowTestCase):
     """, self._StripGraph(gd))
 
   def testPassingList(self):
-    with ops.Graph().as_default():
+    with ops.Graph().as_default() as g:
       a = constant_op.constant(0, name="a")
       b = constant_op.constant(0, name="b")
-      with self.assertRaises(TypeError):
-        control_flow_ops.group([a.op, b.op])
+      control_flow_ops.group([a.op, b.op], name="root")
+    gd = g.as_graph_def()
+    self.assertProtoEquals("""
+      node { name: "a" op: "Const"}
+      node { name: "b" op: "Const"}
+      node { name: "root" op: "NoOp" input: "^a" input: "^b" }
+    """, self._StripGraph(gd))
+
+  def testPassingNonTensors(self):
+    with ops.Graph().as_default():
       with self.assertRaises(TypeError):
         control_flow_ops.group(1, 2)
 
-- 
GitLab


From 177bd25e5d75ab4b21d9aa25e1cba5ff9dbfddc9 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Oct 2017 14:33:17 -0700
Subject: [PATCH 318/573] Nice error messages when using queues / batching in
 eager mode.

PiperOrigin-RevId: 173312134
---
 tensorflow/python/ops/data_flow_ops.py |  5 +++++
 tensorflow/python/training/input.py    | 21 +++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py
index 41dd7f1467..62845a9f8b 100644
--- a/tensorflow/python/ops/data_flow_ops.py
+++ b/tensorflow/python/ops/data_flow_ops.py
@@ -146,7 +146,12 @@ class QueueBase(object):
 
     Raises:
       ValueError: If one of the arguments is invalid.
+      ValueError: If eager execution is enabled.
     """
+    if context.in_eager_mode():
+      raise ValueError(
+          "Queues are not supported in TensorFlow with eager execution. "
+          "Instead, use tf.data to get data into your model.")
     self._dtypes = dtypes
     if shapes is not None:
       if len(shapes) != len(dtypes):
diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py
index 36f97960dd..b999dbedb6 100644
--- a/tensorflow/python/training/input.py
+++ b/tensorflow/python/training/input.py
@@ -27,6 +27,7 @@ import collections
 
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -146,6 +147,10 @@ def input_producer(input_tensor,
   Raises:
     ValueError: If the shape of the input cannot be inferred from the arguments.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   with ops.name_scope(name, "input_producer", [input_tensor]):
     input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")
     element_shape = input_tensor.shape[1:].merge_with(element_shape)
@@ -685,6 +690,10 @@ def _batch(tensors, batch_size, keep_input, num_threads=1, capacity=32,
            allow_smaller_final_batch=False, shared_name=None,
            name=None):
   """Helper function for `batch` and `maybe_batch`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list = _as_tensor_list(tensors)
   with ops.name_scope(name, "batch", list(tensor_list) + [keep_input]) as name:
     tensor_list = _validate(tensor_list)
@@ -718,6 +727,10 @@ def _batch_join(tensors_list, batch_size, keep_input, capacity=32,
                 enqueue_many=False, shapes=None, dynamic_pad=False,
                 allow_smaller_final_batch=False, shared_name=None, name=None):
   """Helper function for `batch_join` and `maybe_batch_join`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list_list = _as_tensor_list_list(tensors_list)
   with ops.name_scope(name, "batch_join",
                       _flatten(tensor_list_list) + [keep_input]) as name:
@@ -748,6 +761,10 @@ def _shuffle_batch(tensors, batch_size, capacity, min_after_dequeue,
                    shapes=None, allow_smaller_final_batch=False,
                    shared_name=None, name=None):
   """Helper function for `shuffle_batch` and `maybe_shuffle_batch`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list = _as_tensor_list(tensors)
   with ops.name_scope(name, "shuffle_batch",
                       list(tensor_list) + [keep_input]) as name:
@@ -788,6 +805,10 @@ def _shuffle_batch_join(tensors_list, batch_size, capacity,
                         allow_smaller_final_batch=False, shared_name=None,
                         name=None):
   """Helper function for `shuffle_batch_join` and `maybe_shuffle_batch_join`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list_list = _as_tensor_list_list(tensors_list)
   with ops.name_scope(name, "shuffle_batch_join",
                       _flatten(tensor_list_list) + [keep_input]) as name:
-- 
GitLab


From dfc7b26b0dc0dd54038a1be3b31b05bd39c1e79f Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Oct 2017 14:42:15 -0700
Subject: [PATCH 319/573] Exception instead of crashing on resource.numpy()

PiperOrigin-RevId: 173313459
---
 tensorflow/python/framework/ops.py                          | 5 +++++
 .../python/kernel_tests/resource_variable_ops_test.py       | 6 ++++++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b3caebce70..94c29c89df 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -615,7 +615,12 @@ class _EagerTensorBase(Tensor):
     Returns:
       A numpy array that may share memory with the Tensor object. Any changes
       to one may be reflected in the other.
+
+    Raises:
+      ValueError: if the type of this Tensor is not representable in numpy.
     """
+    if self.dtype == dtypes.resource:
+      raise ValueError("Resource handles are not convertible to numpy.")
     return self.cpu()._numpy()  # pylint: disable=protected-access
 
   def __array__(self):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 10f9a72c7b..a2a1e1dcd8 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -178,6 +178,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(v.handle.op.colocation_groups(),
                        v.initializer.inputs[1].op.colocation_groups())
 
+  def testHandleNumpy(self):
+    with context.eager_mode():
+      with self.assertRaises(ValueError):
+        resource_variable_ops.ResourceVariable(
+            1.0, name="handle-numpy").handle.numpy()
+
   @test_util.run_in_graph_and_eager_modes()
   def testInitFnDtype(self):
     v = resource_variable_ops.ResourceVariable(
-- 
GitLab


From 8d1a4fa09cb40ee98ecddc99f207f17b05176897 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 14:45:30 -0700
Subject: [PATCH 320/573] Add a MultiHandler that can conditionally apply
 handling logic based on presence of input Tensors.

PiperOrigin-RevId: 173314020
---
 .../python/slim/data/tfexample_decoder.py     | 34 +++++++++++++
 .../slim/data/tfexample_decoder_test.py       | 51 +++++++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index 7a56df9e97..0544404e9e 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -243,6 +243,40 @@ class LookupTensor(Tensor):
     return self._table.lookup(unmapped_tensor)
 
 
+class BackupHandler(ItemHandler):
+  """An ItemHandler that tries two ItemHandlers in order."""
+
+  def __init__(self, handler, backup):
+    """Initializes the BackupHandler handler.
+
+    If the first Handler's tensors_to_item returns a Tensor with no elements,
+    the second Handler is used.
+
+    Args:
+      handler: The primary ItemHandler.
+      backup: The backup ItemHandler.
+
+    Raises:
+      ValueError: if either is not an ItemHandler.
+    """
+    if not isinstance(handler, ItemHandler):
+      raise ValueError('Primary handler is of type %s instead of ItemHandler'
+                       % type(handler))
+    if not isinstance(backup, ItemHandler):
+      raise ValueError('Backup handler is of type %s instead of ItemHandler'
+                       % type(backup))
+    self._handler = handler
+    self._backup = backup
+    super(BackupHandler, self).__init__(handler.keys + backup.keys)
+
+  def tensors_to_item(self, keys_to_tensors):
+    item = self._handler.tensors_to_item(keys_to_tensors)
+    return control_flow_ops.cond(
+        pred=math_ops.equal(math_ops.reduce_prod(array_ops.shape(item)), 0),
+        true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors),
+        false_fn=lambda: item)
+
+
 class SparseTensor(ItemHandler):
   """An ItemHandler for SparseTensors."""
 
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
index 9c5a14d006..d783d4fef4 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
@@ -843,5 +843,56 @@ class TFExampleDecoderTest(test.TestCase):
 
     self.assertAllClose([2, 0, 1], obtained_class_ids)
 
+  def testDecodeExampleWithBackupHandlerLookup(self):
+
+    example1 = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'image/object/class/text':
+                    self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])),
+                'image/object/class/label':
+                    self._EncodedInt64Feature(np.array([42, 10, 900]))
+            }))
+    example2 = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'image/object/class/text':
+                    self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])),
+            }))
+    example3 = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'image/object/class/label':
+                    self._EncodedInt64Feature(np.array([42, 10, 901]))
+            }))
+    # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2
+    table = lookup_ops.index_table_from_tensor(
+        constant_op.constant(['dog', 'guinea pig', 'cat']))
+    keys_to_features = {
+        'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string),
+        'image/object/class/label': parsing_ops.VarLenFeature(dtypes.int64),
+    }
+    backup_handler = tfexample_decoder.BackupHandler(
+        handler=tfexample_decoder.Tensor('image/object/class/label'),
+        backup=tfexample_decoder.LookupTensor('image/object/class/text', table))
+    items_to_handlers = {
+        'labels': backup_handler,
+    }
+    decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
+                                                 items_to_handlers)
+    obtained_class_ids_each_example = []
+    with self.test_session() as sess:
+      sess.run(lookup_ops.tables_initializer())
+      for example in [example1, example2, example3]:
+        serialized_example = array_ops.reshape(
+            example.SerializeToString(), shape=[])
+        obtained_class_ids_each_example.append(
+            decoder.decode(serialized_example)[0].eval())
+
+    self.assertAllClose([42, 10, 900], obtained_class_ids_each_example[0])
+    self.assertAllClose([2, 0, 1], obtained_class_ids_each_example[1])
+    self.assertAllClose([42, 10, 901], obtained_class_ids_each_example[2])
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From b20e0b28eec6245ce734d78cdb26dbf2d92c87ba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 15:07:22 -0700
Subject: [PATCH 321/573] Add transitional and temporary include file to merge
 grpc and grpc::internal namespaces while versions of gRPC are in transition

PiperOrigin-RevId: 173317900
---
 .../contrib/verbs/grpc_verbs_service_impl.cc  |  8 ++---
 .../contrib/verbs/grpc_verbs_service_impl.h   | 11 ++++++-
 tensorflow/core/distributed_runtime/rpc/BUILD | 11 +++++++
 .../rpc/grpc_master_service_impl.cc           | 32 +++++++++----------
 .../rpc/grpc_master_service_impl.h            | 15 +++++----
 .../rpc/grpc_namespace_compat.h               | 32 +++++++++++++++++++
 .../rpc/grpc_worker_service_impl.cc           |  4 +--
 .../rpc/grpc_worker_service_impl.h            |  1 +
 8 files changed, 84 insertions(+), 30 deletions(-)
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h

diff --git a/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc b/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc
index a1fbea57dd..cff765d1e8 100644
--- a/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc
+++ b/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc
@@ -43,21 +43,21 @@ VerbsService::Stub::Stub(
     const std::shared_ptr< ::grpc::ChannelInterface>& channel)
     : channel_(channel),
       rpcmethod_GetRemoteAddress_(grpcVerbsService_method_names[0],
-                                  ::grpc::internal::RpcMethod::NORMAL_RPC,
+                                  ::grpc::RpcMethod::NORMAL_RPC,
                                   channel) {}
 
 ::grpc::Status VerbsService::Stub::GetRemoteAddress(
     ::grpc::ClientContext* context, const GetRemoteAddressRequest& request,
     GetRemoteAddressResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(
+  return ::grpc::BlockingUnaryCall(
       channel_.get(), rpcmethod_GetRemoteAddress_, context, request, response);
 }
 
 VerbsService::AsyncService::AsyncService() {
   for (int i = 0; i < 1; ++i) {
-    AddMethod(new ::grpc::internal::RpcServiceMethod(
+    AddMethod(new ::grpc::RpcServiceMethod(
         grpcVerbsService_method_names[i],
-        ::grpc::internal::RpcMethod::NORMAL_RPC,
+        ::grpc::RpcMethod::NORMAL_RPC,
         nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
diff --git a/tensorflow/contrib/verbs/grpc_verbs_service_impl.h b/tensorflow/contrib/verbs/grpc_verbs_service_impl.h
index 86431ca030..6e2bf86dac 100644
--- a/tensorflow/contrib/verbs/grpc_verbs_service_impl.h
+++ b/tensorflow/contrib/verbs/grpc_verbs_service_impl.h
@@ -28,6 +28,15 @@ limitations under the License.
 #include "tensorflow/contrib/verbs/verbs_service.pb.h"
 
 namespace grpc {
+
+// ensure internal namespace exists
+namespace internal {
+// bring in contents of external namespace
+using namespace ::grpc;
+}  // namespace internal
+// bring in contents of internal namespace
+using namespace internal;
+
 class CompletionQueue;
 class Channel;
 class RpcService;
@@ -61,7 +70,7 @@ class VerbsService GRPC_FINAL {
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    const ::grpc::internal::RpcMethod rpcmethod_GetRemoteAddress_;
+    const ::grpc::RpcMethod rpcmethod_GetRemoteAddress_;
   };
   static std::unique_ptr NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index a8af124e2b..5190288e88 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -182,6 +182,7 @@ cc_library(
     srcs = ["grpc_worker_service_impl.cc"],
     hdrs = ["grpc_worker_service_impl.h"],
     deps = [
+        ":grpc_namespace_compat",
         ":grpc_serialization_traits",
         "//tensorflow/core:worker_proto_cc",
         "//tensorflow/core/distributed_runtime:worker_interface",
@@ -228,12 +229,22 @@ cc_library(
     srcs = ["grpc_master_service_impl.cc"],
     hdrs = ["grpc_master_service_impl.h"],
     deps = [
+        ":grpc_namespace_compat",
         ":grpc_serialization_traits",
         "//tensorflow/core:master_proto_cc",
         "@grpc//:grpc++_unsecure",
     ],
 )
 
+cc_library(
+    name = "grpc_namespace_compat",
+    srcs = [],
+    hdrs = ["grpc_namespace_compat.h"],
+    deps = [
+        "@grpc//:grpc++_unsecure",
+    ],
+)
+
 cc_library(
     name = "grpc_serialization_traits",
     srcs = [],
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc
index 17d0047eb2..d998d51058 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc
@@ -49,74 +49,74 @@ MasterService::Stub::Stub(
     const std::shared_ptr< ::grpc::ChannelInterface>& channel)
     : channel_(channel),
       rpcmethod_CreateSession_(grpcMasterService_method_names[0],
-                               ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                               ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_ExtendSession_(grpcMasterService_method_names[1],
-                               ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                               ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_PartialRunSetup_(grpcMasterService_method_names[2],
-                                 ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                                 ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_RunStep_(grpcMasterService_method_names[3],
-                         ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                         ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_CloseSession_(grpcMasterService_method_names[4],
-                              ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                              ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_ListDevices_(grpcMasterService_method_names[5],
-                             ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                             ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_Reset_(grpcMasterService_method_names[6],
-                       ::grpc::internal::RpcMethod::NORMAL_RPC, channel) {}
+                       ::grpc::RpcMethod::NORMAL_RPC, channel) {}
 
 ::grpc::Status MasterService::Stub::CreateSession(
     ::grpc::ClientContext* context, const CreateSessionRequest& request,
     CreateSessionResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::ExtendSession(
     ::grpc::ClientContext* context, const ExtendSessionRequest& request,
     ExtendSessionResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::PartialRunSetup(
     ::grpc::ClientContext* context, const PartialRunSetupRequest& request,
     PartialRunSetupResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::RunStep(::grpc::ClientContext* context,
                                             const RunStepRequest& request,
                                             RunStepResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
                                    request, response);
 }
 
 ::grpc::Status MasterService::Stub::CloseSession(
     ::grpc::ClientContext* context, const CloseSessionRequest& request,
     CloseSessionResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::ListDevices(
     ::grpc::ClientContext* context, const ListDevicesRequest& request,
     ListDevicesResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::Reset(::grpc::ClientContext* context,
                                           const ResetRequest& request,
                                           ResetResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
                                    request, response);
 }
 
 MasterService::AsyncService::AsyncService() {
   for (int i = 0; i < 7; ++i) {
-    AddMethod(new ::grpc::internal::RpcServiceMethod(
+    AddMethod(new ::grpc::RpcServiceMethod(
         grpcMasterService_method_names[i],
-        ::grpc::internal::RpcMethod::NORMAL_RPC,
+        ::grpc::RpcMethod::NORMAL_RPC,
         nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h
index 412395c526..131de2863f 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "grpc++/impl/codegen/stub_options.h"
 #include "grpc++/impl/codegen/sync_stream.h"
 
+#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h"
 #include "tensorflow/core/protobuf/master.pb.h"
 
@@ -107,13 +108,13 @@ class MasterService final {
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    const ::grpc::internal::RpcMethod rpcmethod_CreateSession_;
-    const ::grpc::internal::RpcMethod rpcmethod_ExtendSession_;
-    const ::grpc::internal::RpcMethod rpcmethod_PartialRunSetup_;
-    const ::grpc::internal::RpcMethod rpcmethod_RunStep_;
-    const ::grpc::internal::RpcMethod rpcmethod_CloseSession_;
-    const ::grpc::internal::RpcMethod rpcmethod_ListDevices_;
-    const ::grpc::internal::RpcMethod rpcmethod_Reset_;
+    const ::grpc::RpcMethod rpcmethod_CreateSession_;
+    const ::grpc::RpcMethod rpcmethod_ExtendSession_;
+    const ::grpc::RpcMethod rpcmethod_PartialRunSetup_;
+    const ::grpc::RpcMethod rpcmethod_RunStep_;
+    const ::grpc::RpcMethod rpcmethod_CloseSession_;
+    const ::grpc::RpcMethod rpcmethod_ListDevices_;
+    const ::grpc::RpcMethod rpcmethod_Reset_;
   };
   static std::unique_ptr NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h b/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h
new file mode 100644
index 0000000000..c178927f5d
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h
@@ -0,0 +1,32 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
+
+// This file is a transitional place-holder until gRPC versions consistently
+// use namespace grpc::internal for library-internal structures
+
+namespace grpc {
+// ensure internal namespace exists
+namespace internal {
+// bring in contents of external namespace
+using namespace ::grpc;
+}  // namespace internal
+// bring in contents of internal namespace
+using namespace internal;
+}  // namespace grpc
+
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc
index 348c6dc98b..80a2f89337 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc
@@ -58,9 +58,9 @@ namespace grpc {
 
 WorkerService::AsyncService::AsyncService() {
   for (int i = 0; i < kGrpcNumWorkerMethods; ++i) {
-    AddMethod(new ::grpc::internal::RpcServiceMethod(
+    AddMethod(new ::grpc::RpcServiceMethod(
         GrpcWorkerMethodName(static_cast(i)),
-        ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
+        ::grpc::RpcMethod::NORMAL_RPC, nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
 }
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
index e9862a61a3..c8a8b5778e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
@@ -26,6 +26,7 @@ limitations under the License.
 #include "grpc++/impl/codegen/sync_stream.h"
 #include "grpc++/support/byte_buffer.h"
 
+#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h"
 #include "tensorflow/core/distributed_runtime/tensor_coding.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
-- 
GitLab


From 488408c2cefcac507b325da4dd779a9015f7b53f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 15:23:58 -0700
Subject: [PATCH 322/573] Removes unnecessary cast and warning in auc
 calculation.

PiperOrigin-RevId: 173320547
---
 tensorflow/python/estimator/BUILD          | 1 -
 tensorflow/python/estimator/canned/head.py | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 9670827e41..13fbfe9f53 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -543,7 +543,6 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
         "//tensorflow/python:nn",
-        "//tensorflow/python:platform",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
         "//tensorflow/python:summary",
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index 1cc82c5055..f26e54ff49 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -40,7 +40,6 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.ops.losses import losses
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.summary import summary
 
@@ -314,9 +313,6 @@ def _predictions_mean(predictions, weights=None, name=None):
 def _auc(labels, predictions, weights=None, curve='ROC', name=None):
   with ops.name_scope(name, 'auc', (predictions, labels, weights)) as scope:
     predictions = math_ops.to_float(predictions, name='predictions')
-    if labels.dtype.base_dtype != dtypes.bool:
-      logging.warning('Casting %s labels to bool.', labels.dtype)
-      labels = math_ops.cast(labels, dtypes.bool)
     if weights is not None:
       weights = weights_broadcast_ops.broadcast_weights(weights, predictions)
     return metrics_lib.auc(
-- 
GitLab


From 64ba163dc8fa1bdf780cbbb67811f9adce05e325 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 15:24:15 -0700
Subject: [PATCH 323/573] Evaluator changes: * Get graph execution working, by
 adding init_variables() and   supporting graph mode in evaluate_on_dataset().
 * Use track_*() instead of add_*() to match Network. * Fill in several doc
 strings. * Detect metric added to two different Evaluators. * Return prefix
 along with metrics from metrics property.

PiperOrigin-RevId: 173320585
---
 tensorflow/contrib/eager/python/BUILD         |  4 +
 tensorflow/contrib/eager/python/evaluator.py  | 99 ++++++++++++++++---
 .../contrib/eager/python/evaluator_test.py    | 51 ++++++++--
 tensorflow/tools/ci_build/ci_sanity.sh        |  1 +
 4 files changed, 132 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index ee2ec79141..bbbf72d632 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -169,6 +169,9 @@ py_library(
     deps = [
         ":datasets",
         ":metrics",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:function",
     ],
@@ -182,6 +185,7 @@ py_test(
         ":evaluator",
         ":metrics",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py
index d757e976ee..67f545e838 100644
--- a/tensorflow/contrib/eager/python/evaluator.py
+++ b/tensorflow/contrib/eager/python/evaluator.py
@@ -22,6 +22,11 @@ import six
 
 from tensorflow.contrib.eager.python import datasets
 from tensorflow.contrib.eager.python import metrics
+from tensorflow.python.eager import context
+from tensorflow.python.eager import function
+from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
 
 
 class Evaluator(object):
@@ -37,7 +42,7 @@ class Evaluator(object):
   the evaluate_on_dataset() method.
 
   Implementers of Evaluators should
-  (a) Call `add_metric()` and/or `add_evaluator()` in __init__().
+  (a) Call `track_metric()` and/or `track_evaluator()` in __init__().
   (b) Override the `call()` method. It will be passed the output of the
       model's `eval_data()` method, and should call its contained metrics
       (treating them as callables) and any child Evaluators (using their
@@ -51,12 +56,36 @@ class Evaluator(object):
     self._model = model
     self._metrics = {}
     self._evaluators = {}
+    if context.in_graph_mode():
+      self.call = function.defun(self.call)
 
   # ---- API for users ----
   def __call__(self, *args, **kwargs):
-    """Update metrics with a minibatch of input examples."""
+    """Update metrics with a minibatch of input examples.
+
+    Args:
+      *args:
+      **kwargs: Arguments representing an input mini-batch of examples to
+        pass to self.model.eval_data().
+
+    Returns:
+      The op to execute or None if executing eagerly.
+    """
     return self.call(self._model.eval_data(*args, **kwargs))
 
+  def init_variables(self):
+    """Return an op for initializing all contained uninitialized variables.
+
+    Only for graph execution. Should be called after variables are created
+    in the first execution of __call__().
+
+    Returns:
+      An op.
+    """
+    assert context.in_graph_mode()
+    return control_flow_ops.group(
+        *[m.init_variables() for _, m in self.metrics])
+
   def all_metric_results(self):  # TODO(josh11b): Add optional summary_writer.
     """Returns dict mapping metric name -> value."""
     results = {}
@@ -69,14 +98,45 @@ class Evaluator(object):
 
   def evaluate_on_dataset(self, dataset, *args, **kwargs):
     """Convenience method for performing an eval on a Dataset."""
+    # TODO(josh11b): Add optional summary_writer.
+    if context.in_graph_mode():
+      # TODO(josh11b): Return an dict of tensors to pass to session.run()
+      # instead of running using the default session here.
+      sess = ops.get_default_session()
+      call_op = self.__call__(dataset.make_one_shot_iterator().get_next(),
+                              *args, **kwargs)
+      init_op = self.init_variables()
+      results_op = self.all_metric_results()
+      sess.run(init_op)
+      try:
+        while True:
+          sess.run(call_op)
+      except errors_impl.OutOfRangeError:
+        pass
+      return sess.run(results_op)
+    # Eager case
     for example in datasets.Iterator(dataset):
       self.__call__(example, *args, **kwargs)
-    # TODO(josh11b): Add optional summary_writer.
     return self.all_metric_results()
 
   # ---- To be implemented by descendants ---
   def call(self, eval_data):
-    """Update metrics using the output of self.model."""
+    """Update metrics using the output of self.model.
+
+    Note: This function is executed as a graph function in graph mode.
+    This means:
+    a) Operations on the same resource are executed in textual order.
+       This should make it easier to do things like add the updated
+       value of a variable to another, for example.
+    b) You don't need to worry about collecting the update ops to execute.
+       All update ops added to the graph by this function will be executed.
+    As a result, code should generally work the same way with graph or
+    eager execution.
+
+    Args:
+      eval_data: The output of self.model.eval_data() on a mini-batch of
+        examples.
+    """
     raise NotImplementedError("Evaluators must define a call member function.")
 
   # ---- For use by descendants ---
@@ -84,10 +144,11 @@ class Evaluator(object):
   def model(self):
     return self._model
 
-  def add_metric(self, metric):
+  def track_metric(self, metric):
     """Add a Metric to be tracked.
 
-    Rule: metrics can only be in one `Evaluator`.
+    Metrics can only be tracked by one `Evaluator`. Metrics must be
+    tracked or they will not appear in `all_metric_results()`.
 
     Args:
       metric: A `Metric` object.
@@ -98,14 +159,15 @@ class Evaluator(object):
     Raises:
       RuntimeError: If called before __init__.
       TypeError: If `metric` is not of the correct type.
-      ValueError: If there is a name collision between Metrics.
+      ValueError: If there is a name collision between Metrics or `metric`
+        has already been added to another `Evaluator`.
     """
     if not hasattr(self, "_metrics"):
       raise RuntimeError(
           "Need to call Evaluator.__init__ before adding metrics")
     if not isinstance(metric, metrics.Metric):
       raise TypeError(
-          "Evaluator.add_metric() passed type %s, not a tfe.metrics.Metric" %
+          "Evaluator.track_metric() passed type %s, not a tfe.metrics.Metric" %
           (type(metric),))
     if metric.name in self._metrics:
       if metric is self._metrics[metric.name]:
@@ -113,10 +175,16 @@ class Evaluator(object):
       raise ValueError(
           "Attempt to add two Metrics with the name '%s' to the same Evaluator "
           "'%s'" % (metric.name, self.name))
+    # pylint: disable=protected-access
+    if hasattr(metric, "_added_to_an_evaluator"):
+      raise ValueError("Metric %s already added to Evaluator %s" %
+                       (metric.name, metric._added_to_an_evaluator))
+    metric._added_to_an_evaluator = self.__class__.__name__
+    # pylint: enable=protected-access
     self._metrics[metric.name] = metric
     return metric
 
-  def add_evaluator(self, prefix, evaluator):
+  def track_evaluator(self, prefix, evaluator):
     """Add a contained `Evaluator`.
 
     This is for delegating to another `Evaluator`, e.g. for when you have a
@@ -141,7 +209,7 @@ class Evaluator(object):
           "Need to call Evaluator.__init__ before adding evaluators")
     if not isinstance(evaluator, Evaluator):
       raise TypeError(
-          "Evaluator.add_evaluator() passed type %s, not a tfe.Evaluator." %
+          "Evaluator.track_evaluator() passed type %s, not a tfe.Evaluator." %
           (type(evaluator),))
     if prefix in self._evaluators:
       if evaluator is self._evaluators[prefix]:
@@ -162,11 +230,12 @@ class Evaluator(object):
 
   @property
   def metrics(self):
+    """Returns a list of (prefix, metric) pairs."""
     m = []
     for metric in six.itervalues(self._metrics):
-      m.append(metric)
-    for evaluator in six.itervalues(self._evaluators):
-      m += evaluator.metrics
+      m.append(("", metric))
+    for prefix, evaluator in six.iteritems(self._evaluators):
+      m += [(prefix + "/" + p, m) for p, m in evaluator.metrics]
     return m
 
 
@@ -196,8 +265,8 @@ class SparseSoftmaxEvaluator(Evaluator):
     super(SparseSoftmaxEvaluator, self).__init__(model)
     # TODO(josh11b): Expand this to include everything from the standard
     # SparseSoftmax Head.
-    self.avg_loss = self.add_metric(metrics.Mean("Avg_Loss"))
-    self.accuracy = self.add_metric(metrics.Accuracy())
+    self.avg_loss = self.track_metric(metrics.Mean("Avg Loss"))
+    self.accuracy = self.track_metric(metrics.Accuracy())
     self.loss_key = loss_key
     self.label_key = label_key
     self.predicted_class_key = predicted_class_key
diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py
index b18463c31a..71e9fa40a8 100644
--- a/tensorflow/contrib/eager/python/evaluator_test.py
+++ b/tensorflow/contrib/eager/python/evaluator_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.contrib.eager.python import evaluator
 from tensorflow.contrib.eager.python import metrics
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 
 
@@ -40,7 +41,7 @@ class SimpleEvaluator(evaluator.Evaluator):
 
   def __init__(self, model):
     super(SimpleEvaluator, self).__init__(model)
-    self.mean = self.add_metric(metrics.Mean("mean"))
+    self.mean = self.track_metric(metrics.Mean("mean"))
 
   def call(self, eval_data):
     self.mean(eval_data)
@@ -50,8 +51,8 @@ class DelegatingEvaluator(evaluator.Evaluator):
 
   def __init__(self, model):
     super(DelegatingEvaluator, self).__init__(model)
-    self.sub = self.add_evaluator("inner", SimpleEvaluator(model))
-    self.mean = self.add_metric(metrics.Mean("outer-mean"))
+    self.sub = self.track_evaluator("inner", SimpleEvaluator(model))
+    self.mean = self.track_metric(metrics.Mean("outer-mean"))
 
   def call(self, eval_data):
     # Keys here come from PrefixLModel, which adds "l_".
@@ -88,13 +89,21 @@ class EvaluatorTest(test.TestCase):
       prefix_count[p] = prefix_count.get(p, 0) + 1
     self.assertEqual({"outer_mean": 2, "mean": 2}, prefix_count)
 
-  def testDataset(self):
+  def testDatasetEager(self):
     e = SimpleEvaluator(IdentityModel())
     ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0])
     results = e.evaluate_on_dataset(ds)
     self.assertEqual(set(["mean"]), set(results.keys()))
     self.assertEqual(6.0, results["mean"].numpy())
 
+  def testDatasetGraph(self):
+    with context.graph_mode(), self.test_session():
+      e = SimpleEvaluator(IdentityModel())
+      ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0])
+      results = e.evaluate_on_dataset(ds)
+      self.assertEqual(set(["mean"]), set(results.keys()))
+      self.assertEqual(6.0, results["mean"])
+
   def testModelProperty(self):
     m = IdentityModel()
     e = SimpleEvaluator(m)
@@ -102,8 +111,34 @@ class EvaluatorTest(test.TestCase):
 
   def testMetricsProperty(self):
     e = DelegatingEvaluator(PrefixLModel())
-    names = set([m.name for m in e.metrics])
-    self.assertEqual(set(["outer-mean", "mean"]), names)
+    names = set([(p, m.name) for p, m in e.metrics])
+    self.assertEqual(set([("", "outer-mean"), ("inner/", "mean")]), names)
+
+  def testSharedMetric(self):
+
+    class MetricArgEvaluator(evaluator.Evaluator):
+
+      def __init__(self, model, m):
+        super(MetricArgEvaluator, self).__init__(model)
+        self.m = self.track_metric(m)
+
+    metric = metrics.Mean("mean")
+    model = IdentityModel()
+    e = MetricArgEvaluator(model, metric)
+    with self.assertRaisesRegexp(ValueError, "already added"):
+      MetricArgEvaluator(model, metric)
+    del e
+
+  def testMetricTrackedTwice(self):
+
+    class MetricTwiceEvaluator(evaluator.Evaluator):
+
+      def __init__(self, model):
+        super(MetricTwiceEvaluator, self).__init__(model)
+        self.m = self.track_metric(metrics.Mean("mean"))
+        self.track_metric(self.m)  # okay to track same metric again
+
+    MetricTwiceEvaluator(IdentityModel())
 
 
 class SparseSoftmaxEvaluatorTest(test.TestCase):
@@ -115,8 +150,8 @@ class SparseSoftmaxEvaluatorTest(test.TestCase):
        e.label_key: [1, 2, 3],
        e.predicted_class_key: [1, 1, 3]})
     results = e.all_metric_results()
-    self.assertEqual(set(["Avg_Loss", "Accuracy"]), set(results.keys()))
-    self.assertEqual(2.0, results["Avg_Loss"].numpy())
+    self.assertEqual(set(["Avg Loss", "Accuracy"]), set(results.keys()))
+    self.assertEqual(2.0, results["Avg Loss"].numpy())
     self.assertEqual(0.75, results["Accuracy"].numpy())
 
 
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 1703cae1e5..26053de4e9 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -95,6 +95,7 @@ do_pylint() {
 "^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\
 "^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\
 "^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\
+"^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden "\
 "^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\
 "^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\
 "^tensorflow/python/keras/_impl/keras/callbacks\.py.*\[E1133.*not-an-iterable"
-- 
GitLab


From 5b9cdb2dcbb057701b0ffb0ec4e0ab555a53390b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 15:48:40 -0700
Subject: [PATCH 324/573] Adding the batch norm adjustment to contrib/layers.

PiperOrigin-RevId: 173324074
---
 .../contrib/layers/python/layers/layers.py    | 20 +++++++++++++++++--
 .../layers/python/layers/layers_test.py       | 20 +++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py
index 29ab281b1a..deeafdf300 100644
--- a/tensorflow/contrib/layers/python/layers/layers.py
+++ b/tensorflow/contrib/layers/python/layers/layers.py
@@ -463,7 +463,8 @@ def batch_norm(inputs,
                scope=None,
                renorm=False,
                renorm_clipping=None,
-               renorm_decay=0.99):
+               renorm_decay=0.99,
+               adjustment=None):
   """Adds a Batch Normalization layer from http://arxiv.org/abs/1502.03167.
 
     "Batch Normalization: Accelerating Deep Network Training by Reducing
@@ -546,6 +547,17 @@ def batch_norm(inputs,
       and should be neither too small (which would add noise) nor too large
       (which would give stale estimates). Note that `decay` is still applied
       to get the means and variances for inference.
+    adjustment: A function taking the `Tensor` containing the (dynamic) shape of
+      the input tensor and returning a pair (scale, bias) to apply to the
+      normalized values (before gamma and beta), only during training. For
+      example,
+        `adjustment = lambda shape: (
+          tf.random_uniform(shape[-1:], 0.93, 1.07),
+          tf.random_uniform(shape[-1:], -0.1, 0.1))`
+      will scale the normalized value by up to 7% up or down, then shift the
+      result by up to 0.1 (with independent scaling and bias for each feature
+      but shared across all examples), and finally apply gamma and/or beta. If
+      `None`, no adjustment is applied.
 
   Returns:
     A `Tensor` representing the output of the operation.
@@ -569,7 +581,10 @@ def batch_norm(inputs,
   #   implementation in normalization_layers.BatchNormalization.
   inputs = ops.convert_to_tensor(inputs)
   rank = inputs.get_shape().ndims
-  possible_to_fuse = batch_weights is None and not renorm and rank in [2, 4]
+  possible_to_fuse = (batch_weights is None and
+                      not renorm and
+                      rank in [2, 4] and
+                      adjustment is None)
   if fused and possible_to_fuse and (
       zero_debias_moving_mean or rank == 2 or
       updates_collections is not ops.GraphKeys.UPDATE_OPS):
@@ -636,6 +651,7 @@ def batch_norm(inputs,
           renorm=renorm,
           renorm_clipping=renorm_clipping,
           renorm_momentum=renorm_decay,
+          adjustment=adjustment,
           name=sc.name,
           _scope=sc,
           _reuse=reuse,
diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py
index 1040ad3ca7..7c77e905f7 100644
--- a/tensorflow/contrib/layers/python/layers/layers_test.py
+++ b/tensorflow/contrib/layers/python/layers/layers_test.py
@@ -2644,6 +2644,26 @@ class BatchNormTest(test.TestCase):
                                        zero_debias_moving_mean=True)
       sess.run(variables_lib.global_variables_initializer())
 
+  def testAdjustmentCreated(self):
+    # Tests that the adjustment is appropriately passed to and used by the core
+    # BN layer.
+    all_adjustments = []
+    def _create_adjustment(shape):
+      adjustments = [array_ops.ones(shape[-1:]), array_ops.zeros(shape[-1:])]
+      all_adjustments.extend(adjustments)
+      return adjustments
+    depth = 8
+    images = array_ops.zeros([10, 5, 5, depth])
+    output = _layers.batch_norm(
+        images,
+        is_training=True,
+        adjustment=_create_adjustment)
+    self.assertListEqual(output.shape.as_list(), images.shape.as_list())
+    self.assertEqual(len(all_adjustments), 2)
+    self.assertListEqual(all_adjustments[0].shape.as_list(), [depth])
+    self.assertListEqual(all_adjustments[1].shape.as_list(), [depth])
+
+
 class LayerNormTest(test.TestCase):
 
   def testUnknownShape(self):
-- 
GitLab


From e74557db4b9f7b57f32118f4bc349f0a56c1e208 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Oct 2017 15:50:05 -0700
Subject: [PATCH 325/573] bool() for eager variables

PiperOrigin-RevId: 173324238
---
 .../python/kernel_tests/resource_variable_ops_test.py       | 5 +++++
 tensorflow/python/ops/resource_variable_ops.py              | 6 ++++++
 tensorflow/python/ops/rnn_cell_impl.py                      | 2 +-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index a2a1e1dcd8..32edc5be7f 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -68,6 +68,11 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(variable.numpy(), 1.0)
       self.assertAllEqual(variable.initialized_value().numpy(), 1.0)
 
+  def testEagerBool(self):
+    with context.eager_mode():
+      v = resource_variable_ops.ResourceVariable(False, name="bool_test")
+      self.assertAllEqual(bool(v), False)
+
   def testAssignVariableDtypeMismatchEager(self):
     with context.eager_mode():
       handle = resource_variable_ops.var_handle_op(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 71e1fb0297..ce81a32924 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -478,6 +478,12 @@ class ResourceVariable(variables.Variable):
         pass  # 'NoneType' object has no attribute 'eager_mode' when context has
               # been unloaded. Will catch other module unloads as well.
 
+  def __nonzero__(self):
+    return self.__bool__()
+
+  def __bool__(self):
+    return bool(self.read_value())
+
   @property
   def dtype(self):
     """The dtype of this variable."""
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 65b0407008..1825e98259 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -698,7 +698,7 @@ class LSTMCell(RNNCell):
     i, j, f, o = array_ops.split(
         value=lstm_matrix, num_or_size_splits=4, axis=1)
     # Diagonal connections
-    if self._use_peepholes and not self._w_f_diag:
+    if self._use_peepholes and self._w_f_diag is None:
       scope = vs.get_variable_scope()
       with vs.variable_scope(
           scope, initializer=self._initializer) as unit_scope:
-- 
GitLab


From fa5921b6cefa3e877348cfa5158143fbc764bc8c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 15:50:41 -0700
Subject: [PATCH 326/573] Exposes precision_at_top_k under tf.metrics.

PiperOrigin-RevId: 173324359
---
 .../contrib/metrics/python/ops/metric_ops.py  |   2 +-
 .../python/kernel_tests/metrics_test.py       | 155 ++++++++++++++----
 tensorflow/python/ops/metrics.py              |   1 +
 tensorflow/python/ops/metrics_impl.py         |  23 ++-
 .../tools/api/golden/tensorflow.metrics.pbtxt |   4 +
 5 files changed, 142 insertions(+), 43 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 5a4c0c4358..675c49dfc3 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -2085,7 +2085,7 @@ def streaming_sparse_precision_at_top_k(top_k_predictions,
   default_name = _at_k_name('precision', class_id=class_id)
   with ops.name_scope(name, default_name,
                       (top_k_predictions, labels, weights)) as name_scope:
-    return metrics_impl._sparse_precision_at_top_k(  # pylint: disable=protected-access
+    return metrics_impl.precision_at_top_k(
         labels=labels,
         predictions_idx=top_k_predictions,
         class_id=class_id,
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index 2472b2a2a6..f21b0dfeab 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -1827,6 +1827,38 @@ def _test_sparse_precision_at_k(predictions,
       test_case.assertEqual(expected, metric.eval())
 
 
+def _test_precision_at_top_k(
+    predictions_idx,
+    labels,
+    expected,
+    k=None,
+    class_id=None,
+    weights=None,
+    test_case=None):
+  with ops.Graph().as_default() as g, test_case.test_session(g):
+    if weights is not None:
+      weights = constant_op.constant(weights, dtypes_lib.float32)
+    metric, update = metrics.precision_at_top_k(
+        predictions_idx=constant_op.constant(predictions_idx, dtypes_lib.int32),
+        labels=labels,
+        k=k,
+        class_id=class_id,
+        weights=weights)
+
+    # Fails without initialized vars.
+    test_case.assertRaises(errors_impl.OpError, metric.eval)
+    test_case.assertRaises(errors_impl.OpError, update.eval)
+    variables.variables_initializer(variables.local_variables()).run()
+
+    # Run per-step op and assert expected values.
+    if math.isnan(expected):
+      test_case.assertTrue(math.isnan(update.eval()))
+      test_case.assertTrue(math.isnan(metric.eval()))
+    else:
+      test_case.assertEqual(expected, update.eval())
+      test_case.assertEqual(expected, metric.eval())
+
+
 def _test_sparse_average_precision_at_k(predictions,
                                         labels,
                                         k,
@@ -1858,6 +1890,7 @@ class SingleLabelSparsePrecisionTest(test.TestCase):
 
   def setUp(self):
     self._predictions = ((0.1, 0.3, 0.2, 0.4), (0.1, 0.2, 0.3, 0.4))
+    self._predictions_idx = [[3], [3]]
     indicator_labels = ((0, 0, 0, 1), (0, 0, 1, 0))
     class_labels = (3, 2)
     # Sparse vs dense, and 1d vs 2d labels should all be handled the same.
@@ -1868,6 +1901,8 @@ class SingleLabelSparsePrecisionTest(test.TestCase):
                 [[class_id] for class_id in class_labels], dtype=np.int64))
     self._test_sparse_precision_at_k = functools.partial(
         _test_sparse_precision_at_k, test_case=self)
+    self._test_precision_at_top_k = functools.partial(
+        _test_precision_at_top_k, test_case=self)
     self._test_sparse_average_precision_at_k = functools.partial(
         _test_sparse_average_precision_at_k, test_case=self)
 
@@ -1877,16 +1912,24 @@ class SingleLabelSparsePrecisionTest(test.TestCase):
       for class_id in (-1, 0, 1, 2, 4):
         self._test_sparse_precision_at_k(
             self._predictions, labels, k=1, expected=NAN, class_id=class_id)
+        self._test_precision_at_top_k(
+            self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id)
 
   def test_at_k1(self):
     for labels in self._labels:
       # Class 3: 1 label, 2 predictions, 1 correct.
       self._test_sparse_precision_at_k(
           self._predictions, labels, k=1, expected=1.0 / 2, class_id=3)
+      self._test_precision_at_top_k(
+          self._predictions_idx, labels, k=1, expected=1.0 / 2, class_id=3)
 
       # All classes: 2 labels, 2 predictions, 1 correct.
       self._test_sparse_precision_at_k(
           self._predictions, labels, k=1, expected=1.0 / 2)
+      self._test_precision_at_top_k(
+          self._predictions_idx, labels, k=1, expected=1.0 / 2)
+      self._test_sparse_average_precision_at_k(
+          self._predictions, labels, k=1, expected=1.0 / 2)
 
 
 class MultiLabelSparsePrecisionTest(test.TestCase):
@@ -1894,6 +1937,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
   def setUp(self):
     self._test_sparse_precision_at_k = functools.partial(
         _test_sparse_precision_at_k, test_case=self)
+    self._test_precision_at_top_k = functools.partial(
+        _test_precision_at_top_k, test_case=self)
     self._test_sparse_average_precision_at_k = functools.partial(
         _test_sparse_average_precision_at_k, test_case=self)
 
@@ -1905,6 +1950,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     labels = np.array([labels_ex1], dtype=np.int64)
     predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3)
     predictions = (predictions_ex1,)
+    predictions_idx_ex1 = (5, 3, 6, 0, 1)
     precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4)
     avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3,
                          (precision_ex1[1] + precision_ex1[3]) / 4)
@@ -1912,6 +1958,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       k = i + 1
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=precision_ex1[i])
+      self._test_precision_at_top_k(
+          (predictions_idx_ex1[:k],), labels, k=k, expected=precision_ex1[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex1[i])
 
@@ -1920,6 +1968,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     labels = np.array([labels_ex2], dtype=np.int64)
     predictions_ex2 = (0.3, 0.5, 0.0, 0.4, 0.0, 0.1, 0.2)
     predictions = (predictions_ex2,)
+    predictions_idx_ex2 = (1, 3, 0, 6, 5)
     precision_ex2 = (0.0 / 1, 0.0 / 2, 1.0 / 3, 2.0 / 4)
     avg_precision_ex2 = (0.0 / 1, 0.0 / 2, precision_ex2[2] / 3,
                          (precision_ex2[2] + precision_ex2[3]) / 4)
@@ -1927,6 +1976,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       k = i + 1
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=precision_ex2[i])
+      self._test_precision_at_top_k(
+          (predictions_idx_ex2[:k],), labels, k=k, expected=precision_ex2[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex2[i])
 
@@ -1942,8 +1993,11 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     ]
     for i in xrange(4):
       k = i + 1
+      predictions_idx = (predictions_idx_ex1[:k], predictions_idx_ex2[:k])
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=streaming_precision[i])
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=k, expected=streaming_precision[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=streaming_average_precision[i])
 
@@ -1969,6 +2023,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     labels = np.array([labels_ex1], dtype=np.int64)
     predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3)
     predictions = (predictions_ex1,)
+    predictions_idx_ex1 = (5, 3, 6, 0, 1)
     precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4)
     avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3,
                          (precision_ex1[1] + precision_ex1[3]) / 4)
@@ -1976,12 +2031,15 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       k = i + 1
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=precision_ex1[i])
+      self._test_precision_at_top_k(
+          (predictions_idx_ex1[:k],), labels, k=k, expected=precision_ex1[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex1[i])
 
   def test_three_labels_at_k5_no_predictions(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sparse_labels = _binary_2d_label_to_2d_sparse_value(
         [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]])
     dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64)
@@ -1991,10 +2049,13 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       for class_id in (-1, 1, 3, 8, 10):
         self._test_sparse_precision_at_k(
             predictions, labels, k=5, expected=NAN, class_id=class_id)
+        self._test_precision_at_top_k(
+            predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
   def test_three_labels_at_k5_no_labels(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sparse_labels = _binary_2d_label_to_2d_sparse_value(
         [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]])
     dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64)
@@ -2004,10 +2065,13 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       for class_id in (0, 4, 6, 9):
         self._test_sparse_precision_at_k(
             predictions, labels, k=5, expected=0.0, class_id=class_id)
+        self._test_precision_at_top_k(
+            predictions_idx, labels, k=5, expected=0.0, class_id=class_id)
 
   def test_three_labels_at_k5(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sparse_labels = _binary_2d_label_to_2d_sparse_value(
         [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]])
     dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64)
@@ -2016,23 +2080,32 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       # Class 2: 2 labels, 2 correct predictions.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=2.0 / 2, class_id=2)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=2.0 / 2, class_id=2)
 
       # Class 5: 1 label, 1 correct prediction.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=1.0 / 1, class_id=5)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=1.0 / 1, class_id=5)
 
       # Class 7: 1 label, 1 incorrect prediction.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=0.0 / 1, class_id=7)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=0.0 / 1, class_id=7)
 
       # All classes: 10 predictions, 3 correct.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=3.0 / 10)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=3.0 / 10)
 
   def test_three_labels_at_k5_some_out_of_range(self):
     """Tests that labels outside the [0, n_classes) range are ignored."""
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sp_labels = sparse_tensor.SparseTensorValue(
         indices=[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0], [1, 1], [1, 2],
                  [1, 3]],
@@ -2043,24 +2116,34 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     # Class 2: 2 labels, 2 correct predictions.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=2.0 / 2, class_id=2)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=2.0 / 2, class_id=2)
 
     # Class 5: 1 label, 1 correct prediction.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=1.0 / 1, class_id=5)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=1.0 / 1, class_id=5)
 
     # Class 7: 1 label, 1 incorrect prediction.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=0.0 / 1, class_id=7)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=0.0 / 1, class_id=7)
 
     # All classes: 10 predictions, 3 correct.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=3.0 / 10)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=3.0 / 10)
 
   def test_3d_nan(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
@@ -2069,12 +2152,16 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     for class_id in (-1, 1, 3, 8, 10):
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=NAN, class_id=class_id)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
   def test_3d_no_labels(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
@@ -2083,12 +2170,16 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     for class_id in (0, 4, 6, 9):
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=0.0, class_id=class_id)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=0.0, class_id=class_id)
 
   def test_3d(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
@@ -2096,80 +2187,84 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     # Class 2: 4 predictions, all correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=4.0 / 4, class_id=2)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=4.0 / 4, class_id=2)
 
     # Class 5: 2 predictions, both correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=2.0 / 2, class_id=5)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=2.0 / 2, class_id=5)
 
     # Class 7: 2 predictions, 1 correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=1.0 / 2, class_id=7)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=1.0 / 2, class_id=7)
 
     # All classes: 20 predictions, 7 correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=7.0 / 20)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=7.0 / 20)
 
   def test_3d_ignore_some(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
 
     # Class 2: 2 predictions, both correct.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=2.0 / 2.0,
-        class_id=2,
+        predictions, labels, k=5, expected=2.0 / 2.0, class_id=2,
+        weights=[[1], [0]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=2.0 / 2.0, class_id=2,
         weights=[[1], [0]])
 
     # Class 2: 2 predictions, both correct.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=2.0 / 2.0,
-        class_id=2,
+        predictions, labels, k=5, expected=2.0 / 2.0, class_id=2,
+        weights=[[0], [1]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=2.0 / 2.0, class_id=2,
         weights=[[0], [1]])
 
     # Class 7: 1 incorrect prediction.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=0.0 / 1.0,
-        class_id=7,
+        predictions, labels, k=5, expected=0.0 / 1.0, class_id=7,
+        weights=[[1], [0]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=0.0 / 1.0, class_id=7,
         weights=[[1], [0]])
 
     # Class 7: 1 correct prediction.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=1.0 / 1.0,
-        class_id=7,
+        predictions, labels, k=5, expected=1.0 / 1.0, class_id=7,
+        weights=[[0], [1]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=1.0 / 1.0, class_id=7,
         weights=[[0], [1]])
 
     # Class 7: no predictions.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=NAN,
-        class_id=7,
+        predictions, labels, k=5, expected=NAN, class_id=7,
+        weights=[[1, 0], [0, 1]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=NAN, class_id=7,
         weights=[[1, 0], [0, 1]])
 
     # Class 7: 2 predictions, 1 correct.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=1.0 / 2.0,
-        class_id=7,
+        predictions, labels, k=5, expected=1.0 / 2.0, class_id=7,
+        weights=[[0, 1], [1, 0]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=1.0 / 2.0, class_id=7,
         weights=[[0, 1], [1, 0]])
 
 
diff --git a/tensorflow/python/ops/metrics.py b/tensorflow/python/ops/metrics.py
index a4e2ef1dad..0465c77691 100644
--- a/tensorflow/python/ops/metrics.py
+++ b/tensorflow/python/ops/metrics.py
@@ -39,6 +39,7 @@
 @@sensitivity_at_specificity
 @@sparse_average_precision_at_k
 @@sparse_precision_at_k
+@@precision_at_top_k
 @@specificity_at_sensitivity
 @@true_negatives
 @@true_negatives_at_thresholds
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 4c3ebb3aae..9273659a77 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -2889,14 +2889,14 @@ def _streaming_sparse_false_positive_at_k(labels,
     return var, state_ops.assign_add(var, batch_total_fp, name='update')
 
 
-def _sparse_precision_at_top_k(labels,
-                               predictions_idx,
-                               k=None,
-                               class_id=None,
-                               weights=None,
-                               metrics_collections=None,
-                               updates_collections=None,
-                               name=None):
+def precision_at_top_k(labels,
+                       predictions_idx,
+                       k=None,
+                       class_id=None,
+                       weights=None,
+                       metrics_collections=None,
+                       updates_collections=None,
+                       name=None):
   """Computes precision@k of the predictions with respect to sparse labels.
 
   Differs from `sparse_precision_at_k` in that predictions must be in the form
@@ -2915,7 +2915,7 @@ def _sparse_precision_at_top_k(labels,
       N >= 1. Commonly, N=1 and predictions has shape [batch size, k].
       The final dimension contains the top `k` predicted class indices.
       [D1, ... DN] must match `labels`.
-    k: Integer, k for @k metric.
+    k: Integer, k for @k metric. Only used for the default op name.
     class_id: Integer class ID for which we want binary metrics. This should be
       in range [0, num_classes], where num_classes is the last dimension of
       `predictions`. If `class_id` is outside this range, the method returns
@@ -2944,6 +2944,7 @@ def _sparse_precision_at_top_k(labels,
   """
   with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id),
                       (predictions_idx, labels, weights)) as scope:
+    labels = _maybe_expand_labels(labels, predictions_idx)
     top_k_idx = math_ops.to_int64(predictions_idx)
     tp, tp_update = _streaming_sparse_true_positive_at_k(
         predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id,
@@ -3038,10 +3039,8 @@ def sparse_precision_at_k(labels,
   """
   with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id),
                       (predictions, labels, weights)) as scope:
-    labels = _maybe_expand_labels(labels, predictions)
-
     _, top_k_idx = nn.top_k(predictions, k)
-    return _sparse_precision_at_top_k(
+    return precision_at_top_k(
         labels=labels,
         predictions_idx=top_k_idx,
         k=k,
diff --git a/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt
index daa3785034..2aab2c4a77 100644
--- a/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt
@@ -68,6 +68,10 @@ tf_module {
     name: "precision_at_thresholds"
     argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "precision_at_top_k"
+    argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
   member_method {
     name: "recall"
     argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-- 
GitLab


From 8e7390ff4e0d9d173df5e193bf90af934e42f193 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Oct 2017 16:13:09 -0700
Subject: [PATCH 327/573] Fix FusedConv2DBiasActivationOp for OIHW filter
 format.

The 'filter' variable wasn't initialized for OIHW filter format.

PiperOrigin-RevId: 173327533
---
 .../kernels/fused_conv2d_bias_activation_op.cc         | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index e4c39739f7..88306094ab 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -445,11 +445,11 @@ void LaunchFusedConv2DBiasActivationOp::
       .set_zero_padding_width(padding_cols / 2);
 
   Tensor maybe_transformed_filter;
-  const Tensor* filter;
-  if (is_int8x4) {
-    // We have already checked filter is OIHW_VECT_I in the constructor.
-    filter = &filter_param;
-  } else if (filter_format == FORMAT_HWIO) {
+  const Tensor* filter = &filter_param;
+  // For qint8, we have already checked filter is OIHW_VECT_I in the
+  // constructor, but we need to test for is_int8x4 so the if block doesn't
+  // generate code for qint8.
+  if (!is_int8x4 && filter_format == FORMAT_HWIO) {
     // Shuffle filter tensor from HWIO to OIHW:
     OP_REQUIRES_OK(ctx, ctx->allocate_temp(
                             DataTypeToEnum::value,
-- 
GitLab


From 56ceca431454635e8ea456cb35f9aeb7f62a8948 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Oct 2017 17:00:23 -0700
Subject: [PATCH 328/573] Disables storing variables in the default variable
 store for eager.

Also disables all functional layers until a non-default store is
implemented.

PiperOrigin-RevId: 173333446
---
 .../kernel_tests/variable_scope_test.py       |  1 -
 tensorflow/python/layers/convolutional.py     | 42 +++++++++
 tensorflow/python/layers/core.py              | 14 +++
 tensorflow/python/layers/core_test.py         | 87 ++++++++-----------
 tensorflow/python/layers/maxout.py            |  5 ++
 tensorflow/python/layers/normalization.py     |  8 +-
 tensorflow/python/layers/pooling.py           | 43 +++++++++
 tensorflow/python/ops/variable_scope.py       |  8 +-
 8 files changed, 153 insertions(+), 55 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 29f583d5ba..efeb25d095 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -67,7 +67,6 @@ class VariableScopeTest(test.TestCase):
     with self.assertRaises(ValueError):
       vs.get_variable("u", [1], reuse=True)  # That fails.
 
-  @test_util.run_in_graph_and_eager_modes()
   def testNamelessStore(self):
     vs = variable_scope._get_default_variable_store()
     vs.get_variable("v1", [2])
diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index c983d3803b..6b371c618f 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -383,7 +383,14 @@ def conv1d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv1D instead.')
   layer = Conv1D(
       filters=filters,
       kernel_size=kernel_size,
@@ -583,7 +590,14 @@ def conv2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv2D instead.')
   layer = Conv2D(
       filters=filters,
       kernel_size=kernel_size,
@@ -785,7 +799,14 @@ def conv3d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv3D instead.')
   layer = Conv3D(
       filters=filters,
       kernel_size=kernel_size,
@@ -1104,7 +1125,14 @@ def separable_conv2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.SeparableConv2D instead.')
   layer = SeparableConv2D(
       filters=filters,
       kernel_size=kernel_size,
@@ -1399,7 +1427,14 @@ def conv2d_transpose(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv2DTranspose instead.')
   layer = Conv2DTranspose(
       filters=filters,
       kernel_size=kernel_size,
@@ -1710,7 +1745,14 @@ def conv3d_transpose(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv3DTranspose instead.')
   layer = Conv3DTranspose(
       filters=filters,
       kernel_size=kernel_size,
diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py
index ef9ff5790c..457bee5cff 100644
--- a/tensorflow/python/layers/core.py
+++ b/tensorflow/python/layers/core.py
@@ -231,7 +231,14 @@ def dense(
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Dense instead.')
   layer = Dense(units,
                 activation=activation,
                 use_bias=use_bias,
@@ -333,7 +340,14 @@ def dropout(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Dropout instead.')
   layer = Dropout(rate, noise_shape=noise_shape, seed=seed, name=name)
   return layer.apply(inputs, training=training)
 
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index d917dcb69c..5184b372ff 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -203,21 +203,15 @@ class DenseTest(test.TestCase):
     self.assertEqual(len(loss_keys), 1)
     self.assertListEqual(dense.losses, loss_keys)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDense(self):
-    inputs = random_ops.random_uniform((5, 3), seed=1)
-    outputs = core_layers.dense(
-        inputs, 2, activation=nn_ops.relu, name='my_dense')
-    if context.in_graph_mode():
+    with self.test_session():
+      inputs = random_ops.random_uniform((5, 3), seed=1)
+      outputs = core_layers.dense(
+          inputs, 2, activation=nn_ops.relu, name='my_dense')
       self.assertEqual(
           len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2)
       self.assertEqual(outputs.op.name, 'my_dense/Relu')
-    else:
-      self.assertEqual(
-          len(_get_variable_dict_from_varstore().values()), 2)
-    self.assertEqual(outputs.get_shape().as_list(), [5, 2])
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseTwice(self):
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
@@ -249,25 +243,21 @@ class DenseTest(test.TestCase):
         vars2 = variables.trainable_variables()
       self.assertEqual(vars1, vars2)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseInitializerFromScope(self):
     with variable_scope.variable_scope(
-        'scope', initializer=init_ops.ones_initializer()):
+        'scope', initializer=init_ops.ones_initializer()), self.test_session():
       inputs = random_ops.random_uniform((5, 3), seed=1)
       core_layers.dense(inputs, 2)
-      self.evaluate(variables.global_variables_initializer())
+      variables.global_variables_initializer().run()
       weights = _get_variable_dict_from_varstore()
       self.assertEqual(len(weights), 2)
       # Check that the matrix weights got initialized to ones (from scope).
-      self.assertAllClose(
-          self.evaluate(weights['scope/dense/kernel'].read_value()),
-          np.ones((3, 2)))
+      self.assertAllClose(weights['scope/dense/kernel'].read_value().eval(),
+                          np.ones((3, 2)))
       # Check that the bias still got initialized to zeros.
-      self.assertAllClose(
-          self.evaluate(weights['scope/dense/bias'].read_value()),
-          np.zeros((2)))
+      self.assertAllClose(weights['scope/dense/bias'].read_value().eval(),
+                          np.zeros((2)))
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseWithCustomGetter(self):
     called = [0]
 
@@ -280,26 +270,26 @@ class DenseTest(test.TestCase):
       core_layers.dense(inputs, 2)
     self.assertEqual(called[0], 2)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseInScope(self):
-    with variable_scope.variable_scope('test'):
-      inputs = random_ops.random_uniform((5, 3), seed=1)
-      core_layers.dense(inputs, 2, name='my_dense')
-      var_dict = _get_variable_dict_from_varstore()
-      var_key = 'test/my_dense/kernel'
-      self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
-    with variable_scope.variable_scope('test1') as scope:
-      inputs = random_ops.random_uniform((5, 3), seed=1)
-      core_layers.dense(inputs, 2, name=scope)
-      var_dict = _get_variable_dict_from_varstore()
-      var_key = 'test1/kernel'
-      self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
-    with variable_scope.variable_scope('test2'):
-      inputs = random_ops.random_uniform((5, 3), seed=1)
-      core_layers.dense(inputs, 2)
-      var_dict = _get_variable_dict_from_varstore()
-      var_key = 'test2/dense/kernel'
-      self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
+    with self.test_session():
+      with variable_scope.variable_scope('test'):
+        inputs = random_ops.random_uniform((5, 3), seed=1)
+        core_layers.dense(inputs, 2, name='my_dense')
+        var_dict = _get_variable_dict_from_varstore()
+        var_key = 'test/my_dense/kernel'
+        self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
+      with variable_scope.variable_scope('test1') as scope:
+        inputs = random_ops.random_uniform((5, 3), seed=1)
+        core_layers.dense(inputs, 2, name=scope)
+        var_dict = _get_variable_dict_from_varstore()
+        var_key = 'test1/kernel'
+        self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
+      with variable_scope.variable_scope('test2'):
+        inputs = random_ops.random_uniform((5, 3), seed=1)
+        core_layers.dense(inputs, 2)
+        var_dict = _get_variable_dict_from_varstore()
+        var_key = 'test2/dense/kernel'
+        self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
 
   @test_util.run_in_graph_and_eager_modes()
   def testComputeOutputShape(self):
@@ -389,17 +379,16 @@ class DropoutTest(test.TestCase):
     self.assertAlmostEqual(0., np_output.min())
     self.assertAllClose(np_output[:, 0, :], np_output[:, 1, :])
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDropout(self):
-    inputs = array_ops.ones((5, 5))
-    dropped = core_layers.dropout(inputs, 0.5, training=True, seed=1)
-    if context.in_graph_mode():
-      self.evaluate(variables.global_variables_initializer())
-    np_output = self.evaluate(dropped)
-    self.assertAlmostEqual(0., np_output.min())
-    dropped = core_layers.dropout(inputs, 0.5, training=False, seed=1)
-    np_output = self.evaluate(dropped)
-    self.assertAllClose(np.ones((5, 5)), np_output)
+    with self.test_session():
+      inputs = array_ops.ones((5, 5))
+      dropped = core_layers.dropout(inputs, 0.5, training=True, seed=1)
+      variables.global_variables_initializer().run()
+      np_output = self.evaluate(dropped)
+      self.assertAlmostEqual(0., np_output.min())
+      dropped = core_layers.dropout(inputs, 0.5, training=False, seed=1)
+      np_output = self.evaluate(dropped)
+      self.assertAllClose(np.ones((5, 5)), np_output)
 
   def testDynamicRate(self):
     with self.test_session() as sess:
diff --git a/tensorflow/python/layers/maxout.py b/tensorflow/python/layers/maxout.py
index 1ea36dbf6a..fa6c8cee97 100644
--- a/tensorflow/python/layers/maxout.py
+++ b/tensorflow/python/layers/maxout.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import gen_array_ops
@@ -50,6 +51,10 @@ def maxout(inputs, num_units, axis=-1, name=None):
    Raises:
     ValueError: if num_units is not multiple of number of features.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'use tf.contrib.layers.MaxOut instead')
   return MaxOut(num_units=num_units, axis=axis, name=name)(inputs)
 
 
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 74246189b5..899be08020 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -717,7 +717,14 @@ def batch_normalization(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.BactchNormalization instead.')
   layer = BatchNormalization(
       axis=axis,
       momentum=momentum,
@@ -749,4 +756,3 @@ def batch_normalization(inputs,
 
 BatchNorm = BatchNormalization
 batch_norm = batch_normalization
-
diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py
index 6245ec5054..ec02ab032d 100644
--- a/tensorflow/python/layers/pooling.py
+++ b/tensorflow/python/layers/pooling.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.layers import base
 from tensorflow.python.layers import utils
@@ -144,7 +145,14 @@ def average_pooling1d(inputs, pool_size, strides,
 
   Returns:
     The output tensor, of rank 3.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.AveragePooling1D instead.')
   layer = AveragePooling1D(pool_size=pool_size,
                            strides=strides,
                            padding=padding,
@@ -206,7 +214,14 @@ def max_pooling1d(inputs, pool_size, strides,
 
   Returns:
     The output tensor, of rank 3.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.MaxPooling1D instead.')
   layer = MaxPooling1D(pool_size=pool_size,
                        strides=strides,
                        padding=padding,
@@ -344,7 +359,14 @@ def average_pooling2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.AveragePooling2D instead.')
   layer = AveragePooling2D(pool_size=pool_size, strides=strides,
                            padding=padding, data_format=data_format,
                            name=name)
@@ -409,7 +431,14 @@ def max_pooling2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.MaxPooling2D instead.')
   layer = MaxPooling2D(pool_size=pool_size, strides=strides,
                        padding=padding, data_format=data_format,
                        name=name)
@@ -560,7 +589,14 @@ def average_pooling3d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.AveragePooling3D instead.')
   layer = AveragePooling3D(pool_size=pool_size, strides=strides,
                            padding=padding, data_format=data_format,
                            name=name)
@@ -629,7 +665,14 @@ def max_pooling3d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.MaxPooling3D instead.')
   layer = MaxPooling3D(pool_size=pool_size, strides=strides,
                        padding=padding, data_format=data_format,
                        name=name)
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 8c5c639b68..08be8574f3 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -724,9 +724,6 @@ class _VariableStore(object):
     if name in self._vars:
       # Here we handle the case when returning an existing variable.
       if reuse is False:
-        if context.in_eager_mode():
-          raise ValueError(
-              "Trying to recreate existing variable: %s" % self._vars[name])
         tb = self._vars[name].op.traceback[::-1]
         # Throw away internal tf entries and only take a few lines.
         tb = [x for x in tb if "tensorflow/python" not in x[0]][:3]
@@ -798,7 +795,10 @@ class _VariableStore(object):
           dtype=variable_dtype,
           validate_shape=validate_shape,
           constraint=constraint)
-    self._vars[name] = v
+    if context.in_graph_mode():
+      # In eager mode we do not want to keep default references to Variable
+      # objects as this will prevent their memory from being released.
+      self._vars[name] = v
     logging.vlog(1, "Created variable %s with shape %s and init %s", v.name,
                  format(shape), initializer)
 
-- 
GitLab


From 171dc9f32182bad58f811c4dedf8e435bb2508d6 Mon Sep 17 00:00:00 2001
From: Mark Daoust 
Date: Tue, 24 Oct 2017 17:01:41 -0700
Subject: [PATCH 329/573] Disambiguate links to "@{$estimators}".

PiperOrigin-RevId: 173333650
---
 tensorflow/docs_src/extend/estimators.md     |  2 +-
 tensorflow/docs_src/extend/index.md          |  2 +-
 tensorflow/docs_src/get_started/estimator.md |  2 +-
 tensorflow/docs_src/get_started/input_fn.md  |  2 +-
 tensorflow/docs_src/tutorials/layers.md      | 10 +++++-----
 tensorflow/docs_src/tutorials/linear.md      |  2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/docs_src/extend/estimators.md b/tensorflow/docs_src/extend/estimators.md
index 5defade7ae..7e6507c584 100644
--- a/tensorflow/docs_src/extend/estimators.md
+++ b/tensorflow/docs_src/extend/estimators.md
@@ -44,7 +44,7 @@ feature columns, input functions, and `train()`/`evaluate()`/`predict()`
 operations. If you've never used tf.estimator before, or need a refresher,
 you should first review the following tutorials:
 
-*   @{$estimator$tf.estimator Quickstart}: Quick introduction to
+*   @{$get_started/estimator$tf.estimator Quickstart}: Quick introduction to
     training a neural network using tf.estimator.
 *   @{$wide$TensorFlow Linear Model Tutorial}: Introduction to
     feature columns, and an overview on building a linear classifier in
diff --git a/tensorflow/docs_src/extend/index.md b/tensorflow/docs_src/extend/index.md
index 5812caaffc..3f30b9a8c2 100644
--- a/tensorflow/docs_src/extend/index.md
+++ b/tensorflow/docs_src/extend/index.md
@@ -14,7 +14,7 @@ TensorFlow:
     add support for your own shared or distributed filesystem.
   * @{$new_data_formats$Custom Data Readers}, which details how to add support
     for your own file and record formats.
-  * @{$estimators$Creating Estimators in tf.contrib.learn}, which explains how
+  * @{$extend/estimators$Creating Estimators in tf.contrib.learn}, which explains how
     to write your own custom Estimator.  For example, you could build your
     own Estimator to implement some variation on standard linear regression.
 
diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md
index 4f3a438d17..ab270d1408 100644
--- a/tensorflow/docs_src/get_started/estimator.md
+++ b/tensorflow/docs_src/get_started/estimator.md
@@ -400,7 +400,7 @@ second sample is *Iris virginica*.
     @{$linear$Large-scale Linear Models with TensorFlow}.
 
 *   To build your own Estimator using tf.estimator APIs, check out
-    @{$estimators$Creating Estimators in tf.estimator}.
+    @{$extend/estimators$Creating Estimators}.
 
 *   To experiment with neural network modeling and visualization in the browser,
     check out [Deep Playground](http://playground.tensorflow.org/).
diff --git a/tensorflow/docs_src/get_started/input_fn.md b/tensorflow/docs_src/get_started/input_fn.md
index 7706c07b1d..9d3af5d96a 100644
--- a/tensorflow/docs_src/get_started/input_fn.md
+++ b/tensorflow/docs_src/get_started/input_fn.md
@@ -11,7 +11,7 @@ median house values.
 The `input_fn` is used to pass feature and target data to the `train`,
 `evaluate`, and `predict` methods of the `Estimator`.
 The user can do feature engineering or pre-processing inside the `input_fn`.
-Here's an example taken from the @{$estimator$tf.estimator Quickstart tutorial}:
+Here's an example taken from the @{$get_started/estimator$tf.estimator Quickstart tutorial}:
 
 ```python
 import numpy as np
diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index 8037c92c73..e808a3677f 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -192,7 +192,7 @@ def cnn_model_fn(features, labels, mode):
 The following sections (with headings corresponding to each code block above)
 dive deeper into the `tf.layers` code used to create each layer, as well as how
 to calculate loss, configure the training op, and generate predictions. If
-you're already experienced with CNNs and @{$estimators$TensorFlow `Estimator`s},
+you're already experienced with CNNs and @{$extend/estimators$TensorFlow `Estimator`s},
 and find the above code intuitive, you may want to skim these sections or just
 skip ahead to ["Training and Evaluating the CNN MNIST
 Classifier"](#training-and-evaluating-the-cnn-mnist-classifier).
@@ -536,8 +536,8 @@ if mode == tf.estimator.ModeKeys.TRAIN:
 ```
 
 > Note: For a more in-depth look at configuring training ops for Estimator model
-> functions, see @{$estimators#defining-the-training-op-for-the-model$"Defining
-> the training op for the model"} in the @{$estimators$"Creating Estimations in
+> functions, see @{$extend/estimators#defining-the-training-op-for-the-model$"Defining
+> the training op for the model"} in the @{$extend/estimators$"Creating Estimations in
 > tf.estimator"} tutorial.
 
 ### Add evaluation metrics
@@ -601,7 +601,7 @@ be saved (here, we specify the temp directory `/tmp/mnist_convnet_model`, but
 feel free to change to another directory of your choice).
 
 > Note: For an in-depth walkthrough of the TensorFlow `Estimator` API, see the
-> tutorial @{$estimators$"Creating Estimators in tf.estimator."}
+> tutorial @{$extend/estimators$"Creating Estimators in tf.estimator."}
 
 ### Set Up a Logging Hook {#set_up_a_logging_hook}
 
@@ -720,7 +720,7 @@ Here, we've achieved an accuracy of 97.3% on our test data set.
 To learn more about TensorFlow Estimators and CNNs in TensorFlow, see the
 following resources:
 
-*   @{$estimators$Creating Estimators in tf.estimator}. An
+*   @{$extend/estimators$Creating Estimators in tf.estimator}. An
     introduction to the TensorFlow Estimator API, which walks through
     configuring an Estimator, writing a model function, calculating loss, and
     defining a training op.
diff --git a/tensorflow/docs_src/tutorials/linear.md b/tensorflow/docs_src/tutorials/linear.md
index 4201a8021b..a6517549c3 100644
--- a/tensorflow/docs_src/tutorials/linear.md
+++ b/tensorflow/docs_src/tutorials/linear.md
@@ -16,7 +16,7 @@ give it a try. This overview uses code samples from the tutorial, but the
 tutorial walks through the code in greater detail.
 
 To understand this overview it will help to have some familiarity
-with basic machine learning concepts, and also with @{$estimator$tf.estimator}.
+with basic machine learning concepts, and also with @{$get_started/estimator$`tf.estimator`}.
 
 [TOC]
 
-- 
GitLab


From 2e2c1753b64a5aaa6a5fa30edf42297bc402f41b Mon Sep 17 00:00:00 2001
From: Igor Saprykin 
Date: Tue, 24 Oct 2017 17:06:15 -0700
Subject: [PATCH 330/573] Add a way to run ops using a step function to
 MonitoredSession.

With this method users have access to a raw Session while getting the benefit of recoverable behavior of MonitoredSession.

PiperOrigin-RevId: 173334319
---
 tensorflow/python/debug/wrappers/framework.py |   6 +-
 .../python/training/monitored_session.py      | 102 ++++++++++-
 .../python/training/monitored_session_test.py | 166 ++++++++++++++++++
 ...ain.-monitored-session.-step-context.pbtxt |  21 +++
 .../tensorflow.train.-monitored-session.pbtxt |   8 +
 ...ular-monitored-session.-step-context.pbtxt |  21 +++
 ...ow.train.-singular-monitored-session.pbtxt |   8 +
 7 files changed, 328 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt

diff --git a/tensorflow/python/debug/wrappers/framework.py b/tensorflow/python/debug/wrappers/framework.py
index 1947d74973..4e243cb6c9 100644
--- a/tensorflow/python/debug/wrappers/framework.py
+++ b/tensorflow/python/debug/wrappers/framework.py
@@ -551,6 +551,10 @@ class BaseDebugWrapperSession(session.SessionInterface):
     return (self._thread_name_filter_pattern and
             not self._thread_name_filter_pattern.match(thread_name))
 
+  def run_step_fn(self, step_fn):
+    return step_fn(
+        monitored_session.MonitoredSession.StepContext(self._sess, self.run))
+
   def partial_run_setup(self, fetches, feeds=None):
     """Sets up the feeds and fetches for partial runs in the session."""
     raise NotImplementedError(
@@ -792,7 +796,7 @@ class NonInteractiveDebugWrapperSession(BaseDebugWrapperSession):
 
   def __init__(self, sess, watch_fn=None, thread_name_filter=None,
                pass_through_operrors=False):
-    """Constructor of DumpingDebugWrapperSession.
+    """Constructor of NonInteractiveDebugWrapperSession.
 
     Args:
       sess: The TensorFlow `Session` object being wrapped.
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index e6162dd34b..2dd2114af0 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -25,6 +25,7 @@ import sys
 import six
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.estimator import util
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -493,6 +494,7 @@ class _MonitoredSession(object):
       self._sess = _RecoverableSession(self._coordinated_creator)
     else:
       self._sess = self._coordinated_creator.create_session()
+    self._stop_requested_in_step_fn = False
 
   @property
   def graph(self):
@@ -520,10 +522,104 @@ class _MonitoredSession(object):
                           options=options,
                           run_metadata=run_metadata)
 
+  def run_step_fn(self, step_fn):
+    """Run ops using a step function.
+
+    Args:
+      step_fn: A function or a method with a single argument of type
+        `StepContext`.  The function may use methods of the argument to
+        perform computations with access to a raw session.
+
+        The returned value of the `step_fn` will be returned from `run_step_fn`,
+        unless a stop is requested.  In that case, the next `should_stop` call
+        will return True.
+
+        Example usage:
+        ```python
+           with tf.Graph().as_default():
+             c = tf.placeholder(dtypes.float32)
+             v = tf.add(c, 4.0)
+             w = tf.add(c, 0.5)
+
+             def step_fn(step_context):
+               a = step_context.session.run(fetches=v, feed_dict={c: 0.5})
+               if a <= 4.5:
+                 step_context.request_stop()
+               return step_context.run_with_hooks(fetches=w, feed_dict={c: 0.1})
+
+             with tf.MonitoredSession() as session:
+               while not session.should_stop():
+                 a = session.run_step_fn(step_fn)
+        ```
+        Hooks interact with the `run_with_hooks()` call inside the `step_fn`
+        as they do with a `MonitoredSession.run` call.
+
+    Returns:
+      Returns the returned value of `step_fn`.
+
+    Raises:
+      StopIteration: if `step_fn` has called `request_stop()`.  It may be
+        caught by `with tf.MonitoredSession()` to close the session.
+      ValueError: if `step_fn` doesn't have a single argument called
+        `step_context`. It may also optionally have `self` for cases when it
+        belongs to an object.
+    """
+    step_fn_arguments = util.fn_args(step_fn)
+    if step_fn_arguments != ('step_context',) and step_fn_arguments != (
+        'self',
+        'step_context',
+    ):
+      raise ValueError(
+          '`step_fn` may either have one `step_context` argument, or'
+          ' `self` and `step_context` arguments if it\'s an instance'
+          ' method. Got {} instead.'.format(step_fn_arguments))
+
+    try:
+      return step_fn(_MonitoredSession.StepContext(self._tf_sess(), self.run))
+    except StopIteration:
+      self._stop_requested_in_step_fn = True
+      raise
+
+  class StepContext(object):
+    """Control flow instrument for the `step_fn` from `run_step_fn()`.
+
+       Users of `step_fn` may perform `run()` calls without running hooks
+       by accessing the `session`.  A `run()` call with hooks may be performed
+       using `run_with_hooks()`.  Computation flow can be interrupted using
+       `request_stop()`.
+    """
+
+    def __init__(self, session, run_with_hooks_fn):
+      """Initializes the `step_context` argument for a `step_fn` invocation.
+
+      Args:
+        session: An instance of `tf.Session`.
+        run_with_hooks_fn: A function for running fetches and hooks.
+      """
+      self._session = session
+      self._run_with_hooks_fn = run_with_hooks_fn
+
+    @property
+    def session(self):
+      return self._session
+
+    def run_with_hooks(self, *args, **kwargs):
+      """Same as `MonitoredSession.run`. Accepts the same arguments."""
+      return self._run_with_hooks_fn(*args, **kwargs)
+
+    def request_stop(self):
+      """Exit the training loop by causing `should_stop()` to return `True`.
+
+         Causes `step_fn` to exit by raising an exception.
+
+      Raises:
+        StopIteration
+      """
+      raise StopIteration('step_fn has requested the iterations to stop.')
+
   def should_stop(self):
-    if self._sess:
-      return self._sess.should_stop()
-    return True
+    return (self._sess is None or self._sess.should_stop() or
+            self._stop_requested_in_step_fn)
 
   def close(self):
     self._close_internal()
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 84d262935a..e729b79425 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -33,10 +33,12 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import debug_pb2
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -1449,6 +1451,170 @@ class MonitoredSessionTest(test.TestCase):
       with monitored_session.MonitoredSession() as session:
         session.close()
 
+  def test_step_fn_example(self):
+    with ops.Graph().as_default():
+      c = array_ops.placeholder(dtypes.float32)
+      v = array_ops.identity(c)
+
+      def step_fn(step_context):
+        value = step_context.run_with_hooks(fetches=v, feed_dict={c: 3.2})
+        return value
+
+      with monitored_session.MonitoredSession() as session:
+        self.assertNear(3.2, session.run_step_fn(step_fn), 0.1)
+
+  def test_step_function_stops(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      with monitored_session.MonitoredSession() as session:
+        self.assertEqual(None, session.run_step_fn(step_fn))
+        self.assertTrue(session.should_stop())
+
+  def test_step_request_stop_without_a_with_block(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      session = monitored_session.MonitoredSession()
+      try:
+        self.assertEqual(None, session.run_step_fn(step_fn))
+      except StopIteration:
+        pass
+      self.assertTrue(session.should_stop())
+
+  def test_step_request_stop_in_a_loop(self):
+    with ops.Graph().as_default():
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      with monitored_session.MonitoredSession() as session:
+        while not session.should_stop():
+          _ = session.run_step_fn(step_fn)
+          self.fail('An exception should be raised on the line above.')
+
+  def test_step_request_stop_with_returning_a_type(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context):
+        del step_context
+        return 'a type'
+
+      with monitored_session.MonitoredSession() as session:
+        self.assertEqual('a type', session.run_step_fn(step_fn))
+
+  def test_step_with_extra_arguments(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context, extra_foo):
+        del step_context, extra_foo
+
+      with monitored_session.MonitoredSession() as session:
+        with self.assertRaisesRegexp(
+            ValueError,
+            '`step_fn` may either have one `step_context` argument'):
+          self.assertEqual(None, session.run_step_fn(step_fn))
+
+  def test_step_fn_belongs_to_a_class(self):
+    with ops.Graph().as_default():
+      c = array_ops.placeholder(dtypes.float32)
+      v = array_ops.identity(c)
+
+      class Model(object):
+
+        def step_fn(self, step_context):
+          value = step_context.run_with_hooks(fetches=v, feed_dict={c: 3.2})
+          return value
+
+      with monitored_session.MonitoredSession() as session:
+        model = Model()
+        self.assertNear(3.2, session.run_step_fn(model.step_fn), 0.1)
+
+  def test_step_fn_belongs_to_a_class_and_has_extra_methods(self):
+    with ops.Graph().as_default():
+
+      class Model(object):
+
+        def step_fn(self, step_context, extra_foo):
+          del step_context, extra_foo
+
+      with monitored_session.MonitoredSession() as session:
+        with self.assertRaisesRegexp(
+            ValueError,
+            '`step_fn` may either have one `step_context` argument'):
+          model = Model()
+          self.assertEqual(None, session.run_step_fn(model.step_fn))
+
+  def test_step_fn_with_hooks(self):
+    with ops.Graph().as_default():
+      var = resource_variable_ops.ResourceVariable(0.0)
+
+      # This test higlights the interaction of hooks with
+      # `Monitoredsession.run_step_fn`.  The order of execution of operations
+      # below is:
+      #   0.  stage_0
+      #   1.  stage_1_0 or stage_1_1 in an undefined order
+      #   2.  stage_2
+
+      stage_0 = state_ops.assign_add(var, 0.3)
+      stage_1_0 = state_ops.assign_add(var, 0.7)
+      # The order of `stage_1_0` and `stage_1_1` is undefined by
+      # `MonitoredSession`, but we should be able to assert when both of them
+      # are complete.  To obtain a consistent result of adding two different
+      # constants to `var`, we rely on a control dependency and
+      # `ResourceVariable`.  Otherwise, it is possible that one of the
+      # additions overwites the result of the other addition.
+      with ops.control_dependencies([stage_1_0]):
+        stage_1_1 = state_ops.assign_add(var, 0.5)
+      stage_2 = state_ops.assign_add(var, 1.1)
+
+      class Hook(session_run_hook.SessionRunHook):
+
+        def __init__(self, testing):
+          self._testing = testing
+
+        def before_run(self, run_context):
+          return session_run_hook.SessionRunArgs(fetches=stage_1_0)
+
+        def after_run(self, run_context, run_values):
+          self._testing.assertNear(0.3 + 0.5 + 0.7,
+                                   run_context.session.run(var), 0.1)
+          self._testing.assertNear(0.3 + 0.5 + 0.7 + 1.1,
+                                   run_context.session.run(stage_2), 0.1)
+
+      def step_fn(step_context):
+        self.assertNear(0.3, step_context.session.run(stage_0), 0.1)
+        return step_context.run_with_hooks(fetches=stage_1_1)
+
+      with monitored_session.MonitoredSession(hooks=[Hook(self)]) as session:
+        self.assertEqual(0.3 + 0.5 + 0.7, session.run_step_fn(step_fn))
+
+  def test_step_fn_with_hooks_and_request_stop(self):
+    with ops.Graph().as_default():
+      trace_the_hook = {'before_run': False, 'after_run': False}
+
+      class Hook(session_run_hook.SessionRunHook):
+
+        def before_run(self, run_context):
+          trace_the_hook['before_run'] = True
+
+        def after_run(self, run_context, run_values):
+          trace_the_hook['after_run'] = True
+
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      with monitored_session.MonitoredSession(hooks=[Hook()]) as session:
+        self.assertEqual(None, session.run_step_fn(step_fn))
+        self.assertTrue(session.should_stop())
+        # `step_context.request_stop()` in a step_fn interrupts the flow of
+        # running the hooks.
+        self.assertFalse(trace_the_hook['before_run'])
+        self.assertFalse(trace_the_hook['after_run'])
+
 
 class SingularMonitoredSessionTest(test.TestCase):
   """Tests SingularMonitoredSession."""
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt
new file mode 100644
index 0000000000..03efe6639e
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.train.MonitoredSession.StepContext"
+tf_class {
+  is_instance: ""
+  is_instance: ""
+  member {
+    name: "session"
+    mtype: ""
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "request_stop"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_with_hooks"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt
index 3a5cc015b4..09b7b3fb53 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: ""
   is_instance: ""
   is_instance: ""
+  member {
+    name: "StepContext"
+    mtype: ""
+  }
   member {
     name: "graph"
     mtype: ""
@@ -19,6 +23,10 @@ tf_class {
     name: "run"
     argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "run_step_fn"
+    argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "should_stop"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
new file mode 100644
index 0000000000..36d8ce7ff8
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.train.SingularMonitoredSession.StepContext"
+tf_class {
+  is_instance: ""
+  is_instance: ""
+  member {
+    name: "session"
+    mtype: ""
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "request_stop"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_with_hooks"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt
index 7caf837cc3..de0f2c1c1a 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: ""
   is_instance: ""
   is_instance: ""
+  member {
+    name: "StepContext"
+    mtype: ""
+  }
   member {
     name: "graph"
     mtype: ""
@@ -23,6 +27,10 @@ tf_class {
     name: "run"
     argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "run_step_fn"
+    argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "should_stop"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From e67f3af48c94c9456c3ff376dc30c82a4bf982cd Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Tue, 24 Oct 2017 17:45:17 -0700
Subject: [PATCH 331/573] Use 'LABEL maintainer=' in Dockerfile (#13961)

* Use 'LABEL maintainer=' in Dockerfile

This fix is a follow up of 13661 to replace `MAINTAINER`
with `LABEL maintainer=` in Dockerfile. The keyword
`MAINTAINER` has long been deprecated and is replaced by `LABEL`,
which is much more flexible and is easily searchable through `docker inspect`.

This fix replaces remaining `MAINTAINER` with `LABEL`.

Signed-off-by: Yong Tang 

* Additional `MAITAINER` -> `LABEL`

Signed-off-by: Yong Tang 
---
 tensorflow/tools/ci_build/Dockerfile.android              | 2 +-
 tensorflow/tools/ci_build/Dockerfile.cmake                | 2 +-
 tensorflow/tools/ci_build/Dockerfile.cpu                  | 2 +-
 tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu    | 2 +-
 tensorflow/tools/ci_build/Dockerfile.gpu                  | 2 +-
 tensorflow/tools/ci_build/Dockerfile.gpu_clang            | 2 +-
 tensorflow/tools/ci_build/Dockerfile.hadoop               | 2 +-
 tensorflow/tools/ci_build/Dockerfile.pi                   | 2 +-
 tensorflow/tools/ci_build/Dockerfile.pi-python3           | 2 +-
 tensorflow/tools/dist_test/Dockerfile                     | 2 +-
 tensorflow/tools/dist_test/Dockerfile.local               | 2 +-
 tensorflow/tools/dist_test/local/Dockerfile               | 2 +-
 tensorflow/tools/dist_test/server/Dockerfile              | 2 +-
 tensorflow/tools/dist_test/server/Dockerfile.test         | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 | 2 +-
 tensorflow/tools/gcs_test/Dockerfile                      | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.android b/tensorflow/tools/ci_build/Dockerfile.android
index facff47621..99a69d7b43 100644
--- a/tensorflow/tools/ci_build/Dockerfile.android
+++ b/tensorflow/tools/ci_build/Dockerfile.android
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach 
+LABEL maintainer="Jan Prach "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake
index 9013dc012d..37ba24d65a 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cmake
+++ b/tensorflow/tools/ci_build/Dockerfile.cmake
@@ -14,7 +14,7 @@
 # ==============================================================================
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu b/tensorflow/tools/ci_build/Dockerfile.cpu
index 206108930a..57a854a9df 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.cpu
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach 
+LABEL maintainer="Jan Prach "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
index b914f51918..eb9d0d4dd0 100644
--- a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
@@ -1,6 +1,6 @@
 FROM debian:jessie
 
-MAINTAINER Jan Prach 
+LABEL maintainer="Jan Prach "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index 5d18295f68..2d46ccb6b1 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu14.04
 
-MAINTAINER Jan Prach 
+LABEL maintainer="Jan Prach "
 
 # In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to
 # /usr/local/cuda
diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu_clang b/tensorflow/tools/ci_build/Dockerfile.gpu_clang
index c4342d17f5..0ecd8c75e0 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu_clang
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu_clang
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu14.04
 
-MAINTAINER Ilya Biryukov 
+LABEL maintainer="Ilya Biryukov "
 
 # In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to
 # /usr/local/cuda
diff --git a/tensorflow/tools/ci_build/Dockerfile.hadoop b/tensorflow/tools/ci_build/Dockerfile.hadoop
index 489493c26e..6010aedb33 100644
--- a/tensorflow/tools/ci_build/Dockerfile.hadoop
+++ b/tensorflow/tools/ci_build/Dockerfile.hadoop
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jonathan Hseu 
+LABEL maintainer="Jonathan Hseu "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi b/tensorflow/tools/ci_build/Dockerfile.pi
index 2fddd6a2c0..75ef30d32b 100644
--- a/tensorflow/tools/ci_build/Dockerfile.pi
+++ b/tensorflow/tools/ci_build/Dockerfile.pi
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach 
+LABEL maintainer="Jan Prach "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3
index 18b131ea19..b1c648ba30 100644
--- a/tensorflow/tools/ci_build/Dockerfile.pi-python3
+++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach 
+LABEL maintainer="Jan Prach "
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/dist_test/Dockerfile b/tensorflow/tools/dist_test/Dockerfile
index cd64e2c518..2a7605bbc9 100644
--- a/tensorflow/tools/dist_test/Dockerfile
+++ b/tensorflow/tools/dist_test/Dockerfile
@@ -20,7 +20,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 RUN apt-get update
 RUN apt-get install -y \
diff --git a/tensorflow/tools/dist_test/Dockerfile.local b/tensorflow/tools/dist_test/Dockerfile.local
index 7a896ab611..795aeee1b5 100644
--- a/tensorflow/tools/dist_test/Dockerfile.local
+++ b/tensorflow/tools/dist_test/Dockerfile.local
@@ -19,7 +19,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 # Pick up some TF dependencies.
 RUN apt-get update && apt-get install -y \
diff --git a/tensorflow/tools/dist_test/local/Dockerfile b/tensorflow/tools/dist_test/local/Dockerfile
index 96846f6564..383c3c2f4c 100644
--- a/tensorflow/tools/dist_test/local/Dockerfile
+++ b/tensorflow/tools/dist_test/local/Dockerfile
@@ -1,6 +1,6 @@
 FROM jpetazzo/dind
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 RUN apt-get update
 
diff --git a/tensorflow/tools/dist_test/server/Dockerfile b/tensorflow/tools/dist_test/server/Dockerfile
index fabc8a7105..1359428f11 100644
--- a/tensorflow/tools/dist_test/server/Dockerfile
+++ b/tensorflow/tools/dist_test/server/Dockerfile
@@ -19,7 +19,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y \
diff --git a/tensorflow/tools/dist_test/server/Dockerfile.test b/tensorflow/tools/dist_test/server/Dockerfile.test
index 908af8af9b..ce7e783a1a 100644
--- a/tensorflow/tools/dist_test/server/Dockerfile.test
+++ b/tensorflow/tools/dist_test/server/Dockerfile.test
@@ -19,7 +19,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index 4558bc5293..64ebc4607a 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
 
-MAINTAINER Gunhan Gulsoy 
+LABEL maintainer="Gunhan Gulsoy "
 
 # It is possible to override these for releases.
 ARG TF_BRANCH=master
diff --git a/tensorflow/tools/gcs_test/Dockerfile b/tensorflow/tools/gcs_test/Dockerfile
index 5af753226f..69b554047b 100644
--- a/tensorflow/tools/gcs_test/Dockerfile
+++ b/tensorflow/tools/gcs_test/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai 
+LABEL maintainer="Shanqing Cai "
 
 RUN apt-get update
 RUN apt-get install -y \
-- 
GitLab


From 7ff50995aeffff8f534f6d9758a98ca9418e6816 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena 
Date: Tue, 24 Oct 2017 18:03:26 -0700
Subject: [PATCH 332/573] Make Iterators saveable. Add
 tf.contrib.data.make_saveable_from_iterator(iterator) that builds a
 SaveableObject for an iterator so it can be saved/restored using tf.Saver.

PiperOrigin-RevId: 173340191
---
 tensorflow/contrib/data/BUILD                 |   1 +
 tensorflow/contrib/data/__init__.py           |   2 +
 .../contrib/data/python/kernel_tests/BUILD    |   2 +
 .../kernel_tests/range_dataset_op_test.py     | 133 ++++++++++++++++++
 tensorflow/contrib/data/python/ops/BUILD      |  13 ++
 .../contrib/data/python/ops/iterator_ops.py   |  77 ++++++++++
 6 files changed, 228 insertions(+)
 create mode 100644 tensorflow/contrib/data/python/ops/iterator_ops.py

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index ee96269a73..b485d78f5c 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -10,6 +10,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:iterator_ops",
         "//tensorflow/contrib/data/python/ops:readers",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
         "//tensorflow/python:util",
diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 7ff26e087b..e0aab1cd83 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -27,6 +27,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@enumerate_dataset
 @@group_by_window
 @@ignore_errors
+@@make_saveable_from_iterator
 @@read_batch_features
 @@unbatch
 @@rejection_resample
@@ -49,6 +50,7 @@ from tensorflow.contrib.data.python.ops.dataset_ops import get_single_element
 from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset
 from tensorflow.contrib.data.python.ops.error_ops import ignore_errors
 from tensorflow.contrib.data.python.ops.grouping import group_by_window
+from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator
 from tensorflow.contrib.data.python.ops.readers import FixedLengthRecordDataset
 from tensorflow.contrib.data.python.ops.readers import read_batch_features
 from tensorflow.contrib.data.python.ops.readers import SqlDataset
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index b3175e3e56..96447abd7c 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -245,6 +245,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:iterator_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
@@ -257,6 +258,7 @@ py_test(
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:training",
         "//tensorflow/python:variables",
         "//tensorflow/python/data/ops:iterator_ops",
     ],
diff --git a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
index c944eb4a49..f59ac760dc 100644
--- a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
@@ -21,6 +21,7 @@ import os
 
 from tensorflow.contrib.data.python.ops import dataset_ops
 from tensorflow.contrib.data.python.ops import enumerate_ops
+from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -34,6 +35,7 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
+from tensorflow.python.training import saver as saver_lib
 
 
 class RangeDatasetTest(test.TestCase):
@@ -259,6 +261,137 @@ class RangeDatasetTest(test.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  def testSaveRestoreUsingSaverFromMetaGraph(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      ops.add_to_collection("iterator_ops", init_op)
+      ops.add_to_collection("iterator_ops", get_next)
+      saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection
+      # so that it can be automatically picked up by the Saver.
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+      saver = saver_lib.Saver()
+      return init_op, get_next, saver
+
+    start = 2
+    stop = 10
+    break_point = 5
+    path = self._iterator_checkpoint_prefix()
+    meta_filename = path + ".meta"
+
+    # Execute input pipeline for a few steps and save iterator state.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop)
+      with self.test_session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        saver.save(sess, path)
+
+    # Build the saver from the MetaGraph using import_meta_graph and
+    # check that the iterator state is restored.
+    with ops.Graph().as_default() as g:
+      saver = saver_lib.import_meta_graph(meta_filename)
+      init_op, get_next = ops.get_collection("iterator_ops")
+      with self.test_session(graph=g) as sess:
+        saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir()))
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreUsingBuiltSaver(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      ops.add_to_collection("iterator_ops", init_op)
+      ops.add_to_collection("iterator_ops", get_next)
+      # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection
+      # so that it can be automatically picked up by the Saver.
+      saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+      saver = saver_lib.Saver()
+      return init_op, get_next, saver
+
+    start = 2
+    stop = 10
+    stop_new = 15
+    break_point = 5
+    path = self._iterator_checkpoint_prefix()
+
+    # Execute input pipeline for a few steps and save iterator state.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop)
+      with self.test_session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        saver.save(sess, path)
+
+    # Manually build a modified Graph and Saver instead of importing
+    # MetaGraph and verify that original iterator state gets restored.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop_new)
+      with self.test_session(graph=g) as sess:
+        saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir()))
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreUsingSaverThenInit(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      ops.add_to_collection("iterator_ops", init_op)
+      ops.add_to_collection("iterator_ops", get_next)
+      # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection
+      # so that it can be automatically picked up by the Saver.
+      saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+      saver = saver_lib.Saver()
+      return init_op, get_next, saver
+
+    start = 2
+    stop = 10
+    stop_new = 15
+    break_point = 5
+    path = self._iterator_checkpoint_prefix()
+
+    # Execute input pipeline for a few steps and save iterator state.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop)
+      with self.test_session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        saver.save(sess, path)
+
+    # Restore iterator state call and then call init_op for the iterator and
+    # verify that the new iterator hides the restored iterator.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop_new)
+      with self.test_session(graph=g) as sess:
+        saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir()))
+        sess.run(init_op)
+        for i in range(start, stop_new):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
   def testRestoreWithoutBuildingDatasetGraph(self):
 
     def _build_graph(start, stop, num_epochs):
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 2a9b41d6df..b17b02ee35 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -20,6 +20,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "iterator_ops",
+    srcs = [
+        "iterator_ops.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+    ],
+)
+
 py_library(
     name = "readers",
     srcs = [
diff --git a/tensorflow/contrib/data/python/ops/iterator_ops.py b/tensorflow/contrib/data/python/ops/iterator_ops.py
new file mode 100644
index 0000000000..d736029fb0
--- /dev/null
+++ b/tensorflow/contrib/data/python/ops/iterator_ops.py
@@ -0,0 +1,77 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Iterator ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.training import saver
+
+
+def make_saveable_from_iterator(iterator):
+  """Returns a SaveableObject for saving/restore iterator state using Saver.
+
+  Args:
+    iterator: Iterator.
+
+  For example:
+
+  ```python
+  with tf.Graph().as_default():
+    ds = tf.data.Dataset.range(10)
+    iterator = ds.make_initializable_iterator()
+    # Build the iterator SaveableObject.
+    saveable_obj = tf.contrib.data.make_saveable_from_iterator(iterator)
+    # Add the SaveableObject to the SAVEABLE_OBJECTS collection so
+    # it can be automatically saved using Saver.
+    tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+    saver = tf.train.Saver()
+
+    while continue_training:
+      ... Perform training ...
+      if should_save_checkpoint:
+        saver.save()
+  ```
+
+  Note: When restoring the iterator, the existing iterator state is completely
+  discarded. This means that any changes you may have made to the Dataset
+  graph will be discarded as well! This includes the new Dataset graph
+  that you may have built during validation. So, while running validation,
+  make sure to run the initializer for the validation input pipeline after
+  restoring the checkpoint.
+
+  Note: Not all iterators support checkpointing yet. Attempting to save the
+  state of an unsupported iterator will throw an error.
+  """
+  return _Saveable(iterator._iterator_resource)  # pylint: disable=protected-access
+
+
+class _Saveable(saver.BaseSaverBuilder.SaveableObject):
+  """SaveableObject for saving/restoring iterator state."""
+
+  def __init__(self, iterator_resource):
+    serialized_iterator = gen_dataset_ops.serialize_iterator(iterator_resource)
+    specs = [
+        saver.BaseSaverBuilder.SaveSpec(serialized_iterator, "",
+                                        iterator_resource.name + "-state")
+    ]
+    super(_Saveable, self).__init__(iterator_resource, specs,
+                                    iterator_resource.name)
+
+  def restore(self, restored_tensors, unused_restored_shapes):
+    with ops.colocate_with(self.op):
+      return gen_dataset_ops.deserialize_iterator(self.op, restored_tensors[0])
-- 
GitLab


From f1ecdd6ea3eec5f75ba47676dbfa3c283b4e172a Mon Sep 17 00:00:00 2001
From: Michael Case 
Date: Tue, 24 Oct 2017 18:07:59 -0700
Subject: [PATCH 333/573] Write common android build configs to .bazelrc

Add --config=android_arm and --config=android_arm64 Bazel configs
to add options needed for Android Bazel builds.

PiperOrigin-RevId: 173340664
---
 configure.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 9da49b628d..ea3f598f3d 100644
--- a/configure.py
+++ b/configure.py
@@ -963,6 +963,19 @@ def set_monolithic():
   write_to_bazelrc('build --define framework_shared_object=true')
 
 
+def create_android_bazelrc_configs():
+  # Flags for --config=android
+  write_to_bazelrc('build:android --crosstool_top=//external:android/crosstool')
+  write_to_bazelrc(
+      'build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain')
+  # Flags for --config=android_arm
+  write_to_bazelrc('build:android_arm --config=android')
+  write_to_bazelrc('build:android_arm --cpu=armeabi-v7a')
+  # Flags for --config=android_arm64
+  write_to_bazelrc('build:android_arm64 --config=android')
+  write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a')
+
+
 def main():
   # Make a copy of os.environ to be clear when functions and getting and setting
   # environment variables.
@@ -1032,7 +1045,7 @@ def main():
   set_cc_opt_flags(environ_cp)
   set_mkl()
   set_monolithic()
-
+  create_android_bazelrc_configs()
 
 if __name__ == '__main__':
   main()
-- 
GitLab


From e384e28a97bc9f1da1402c93087e0394e5a0168c Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Tue, 24 Oct 2017 18:55:42 -0700
Subject: [PATCH 334/573] Re-land: Dump the computation's SessionModule as part
 of the tf_compile rule.

Nondeterminism in the SessionModule proto dumped by aot/compile.cc was
causing problems for some users. Re-landed with the SessionModule proto
being generated in a different genrule (so as not to disturb existing
users), and with more determinism in the dumped proto.

PiperOrigin-RevId: 173344189
---
 tensorflow/compiler/aot/compile.cc    | 10 +++++++---
 tensorflow/compiler/aot/flags.cc      |  5 ++---
 tensorflow/compiler/aot/flags.h       |  2 +-
 tensorflow/compiler/aot/tfcompile.bzl | 28 +++++++++++++++++++++++++++
 4 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index eac8da0ab1..2b8cc6024c 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -97,11 +97,15 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config,
   TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client,
                                           &computation,
                                           &compile_result->has_context_arg));
-  if (!flags.debug_dir.empty()) {
+  if (!flags.out_session_module.empty()) {
     TF_ASSIGN_OR_RETURN(std::unique_ptr module,
                         computation.Snapshot());
-    string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb");
-    TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module));
+    // Serialize the SessionModule deterministically so that all the outputs of
+    // a tf_library genrule are deterministic.
+    string proto;
+    TF_RET_CHECK(SerializeToStringDeterministic(*module, &proto));
+    TF_RETURN_IF_ERROR(
+        WriteStringToFile(Env::Default(), flags.out_session_module, proto));
   }
   xla::cpu::CpuAotCompilationOptions aot_opts(
       flags.target_triple, flags.target_cpu, flags.target_features,
diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc
index 5aff10346f..7c2f27e550 100644
--- a/tensorflow/compiler/aot/flags.cc
+++ b/tensorflow/compiler/aot/flags.cc
@@ -33,9 +33,6 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) {
        "fetch nodes will be dumped to stdout in a comma-separated list.  "
        "Typically used to format arguments for other tools, e.g. "
        "freeze_graph."},
-      {"debug_dir", &flags->debug_dir,
-       "Specifies a directory to dump debugging information, including "
-       "rewritten graphs and the XLA HLO module."},
       // Flags controlling the XLA ahead-of-time compilation, that correspond to
       // the fields of xla::cpu::CpuAotCompilationOptions.
       //
@@ -64,6 +61,8 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) {
        "namespaces are given, within the global namespace."},
       {"out_object", &flags->out_object, "Output object file name."},
       {"out_header", &flags->out_header, "Output header file name."},
+      {"out_session_module", &flags->out_session_module,
+       "Output session module proto."},
       {"gen_name_to_index", &flags->gen_name_to_index,
        "Generate name-to-index data for Lookup{Arg,Result}Index methods."},
       {"gen_program_shape", &flags->gen_program_shape,
diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h
index 3246dbf95c..3519659e3a 100644
--- a/tensorflow/compiler/aot/flags.h
+++ b/tensorflow/compiler/aot/flags.h
@@ -29,7 +29,6 @@ struct MainFlags {
   string graph;
   string config;
   bool dump_fetch_nodes = false;
-  string debug_dir;
   string target_triple;
   string target_cpu;
   string target_features;
@@ -37,6 +36,7 @@ struct MainFlags {
   string cpp_class;
   string out_object;
   string out_header;
+  string out_session_module;
 
   // C++ codegen options
   bool gen_name_to_index = false;
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 4888760acd..2adb1dc65e 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -165,6 +165,34 @@ def tf_library(name, graph, config,
       tags=tags,
   )
 
+  # Rule that runs tfcompile to produce the SessionModule proto, useful for
+  # debugging.  TODO(b/64813587): Once the SessionModule proto is
+  # deterministic, move this into the main rule above.
+  session_module_pb = name + "_session_module.pb"
+  native.genrule(
+      name=(name + "_session_module"),
+      srcs=[
+          tfcompile_graph,
+          config,
+      ],
+      outs=[
+          session_module_pb,
+      ],
+      cmd=("$(location " + tfcompile_tool + ")" +
+           " --graph=$(location " + tfcompile_graph + ")" +
+           " --config=$(location " + config + ")" +
+           " --entry_point=" + ep +
+           " --cpp_class=" + cpp_class +
+           " --target_triple=" + target_llvm_triple() +
+           " --out_session_module=$(@D)/" + session_module_pb +
+           " " + (tfcompile_flags or "")),
+      tools=[tfcompile_tool],
+      visibility=visibility,
+      testonly=testonly,
+      local=1,
+      tags=tags,
+  )
+
   # The cc_library rule packaging up the header and object file, and needed
   # kernel implementations.
   need_xla_data_proto = (tfcompile_flags and
-- 
GitLab


From 355e25ebcab64e833dfc987638c3e6c79d838266 Mon Sep 17 00:00:00 2001
From: Benoit Steiner 
Date: Tue, 24 Oct 2017 19:47:46 -0700
Subject: [PATCH 335/573] Merge changes from github. END_PUBLIC

---
Commit 9f8523640 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Update ops-related pbtxt files.

PiperOrigin-RevId: 173145770

---
Commit 01b6b0638 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Cut tracing memory cost

PiperOrigin-RevId: 173144626

---
Commit 5e23e0e67 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
[XLA] Erase cloned instructions on the fly when merging fusion nodes.

This avoids the awkward situation where an RNG which is clearly eligible for fusion becomes ineligible mid-fusion because it suddenly has an extra (dead) user.

PiperOrigin-RevId: 173141716

---
Commit 1038927c0 authored by Saurabh Saxena
Committed by TensorFlower Gardener:
Add SerializeIterator op that serializes an IteratorResource into a variant tensor.
Add DeserializeIterator op that builds IteratorResource from a variant tensor.
Move BundleReaderWrapper and BundleWriterWrapper from dataset.h to iterator_ops.cc.
Add generic key-value store interfaces IteratorStateReader and IteratorStateWriter for reading/writing state of iterators.
Get rid of IteratorBundleReader and IteratorBundleWriter.

PiperOrigin-RevId: 173140858

---
Commit 57f3e529d authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Internal change

PiperOrigin-RevId: 173136642

---
Commit 0e56ffb7b authored by Shanqing Cai
Committed by TensorFlower Gardener:
Fix breakages in OSS builds

See example breakages logs at:
http://ci.tensorflow.org/job/tensorflow-cl-cpu-python3-pip/10847/console
http://ci.tensorflow.org/job/tensorflow-cl-gpu/11008/console

1. CL/172477381 added the no_oss tag to tests with oss_serial tags, which broke the logic of OSS_SERIAL tests in pip.sh and run_pip_test.sh. This CL fixes that.

2. The nccl_kernels BUILD target in contrib/nccl/BUILD was missing some dependencies. This CL adds the missing ones.

Fixes: #13918
PiperOrigin-RevId: 173133914

---
Commit 3ed049b67 authored by Alexandre Passos
Committed by TensorFlower Gardener:
Allows calling keras layers in eager mode.

PiperOrigin-RevId: 173129805

---
Commit 4ec6f2b07 authored by Alexandre Passos
Committed by TensorFlower Gardener:
Switching contrib.summaries API to be context-manager-centric

PiperOrigin-RevId: 173129793

---
Commit 03b02ffc9 authored by Justine Tunney
Committed by TensorFlower Gardener:
Put Bazel mirror URLs first

PiperOrigin-RevId: 173127955

---
Commit 46ab25e4d authored by David Majnemer
Committed by TensorFlower Gardener:
[XLA] Add support for convolutions with no spatial dimensions

PiperOrigin-RevId: 173126950

---
Commit fc56349b7 authored by Derek Murray
Committed by TensorFlower Gardener:
[tf.data] Convert dataset arguments to tensors as early as possible.

This change raises a `TypeError` earlier if (for example) the `batch_size`
argument to `Dataset.batch()` has the incorrect type.

PiperOrigin-RevId: 173126678

---
Commit 4f7503a87 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
K-FAC: Support for registering multiple minibatches with register_fully_connected()

PiperOrigin-RevId: 173121735

---
Commit 2845bfcd6 authored by Tim Harley
Committed by TensorFlower Gardener:
Avoid listing all modified Enter/RefEnter nodes on INFO, use VLOG(1) instead.

Leave a single, simple, message on INFO.

PiperOrigin-RevId: 173121726

---
Commit 434695921 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
K-FAC: _check_registration() supports multiple towers.

PiperOrigin-RevId: 173115870

---
Commit 670dddf4a authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Multi-minibatch support for
tf.contrib.kfac.fisher_blocks.FullyConnectedKFACBasicFB.

PiperOrigin-RevId: 173109677

---
Commit dc13a8e2f authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Fix import of meta graphs with partitioned variables into a scope.

Saver inspects SliceInfo to decide the variable name when creating a
checkpoint. Before this fix even if a partitioned variable ("weights")
was imported into a scope "a" it would still be checkpointed as ("weights")
instead of ("a/weights") since import_scoped_meta_graph was not adjusting
the SliceInfo.

WARNING: if you use import_meta_graph on graphs with partitioned_variables WITH an import_scope argument AND then create a Saver to write/read checkpoints this change
may break your checkpoint loading.
PiperOrigin-RevId: 173105796

---
Commit eea089bdb authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
K-FAC: Multi-tower support for ConvDiagonalFB.

PiperOrigin-RevId: 173105412

---
Commit 9b9cbbe2a authored by Yong Tang
Committed by Vijay Vasudevan:
Add int64 Tperm type support for `Transpose` (#13909)

* Add int64 Tperm type support for `Transpose`

This fix adds int64 Tperm support for `Transpose`. In
`array_ops.cc`, `Transpose` and `ConjugateTranspose`
have been specified as accepting int32 and int64 perm
types. However, only int32 kernels has been registered.

This fix adds the int64 perm support by removing
the constraint on Tperm, resolve the type at runtime,
and copying the data type accordingly to correctly handle
the int64/int32 types.

Additional tests have been added as well.

Signed-off-by: Yong Tang 

* Add test cases for int64 of perm in Transpose.

Signed-off-by: Yong Tang 

* Add namespace to hide PermutationHelper

Signed-off-by: Yong Tang 

* Enable use_gpu=True for perm type test.

Signed-off-by: Yong Tang 

* extra // namespace annotation

* Adding a comment about int32 casting that should be safe.

Permutations only contain values that refer to dimensions, and the maximum number of dimensions we have is 254, so an int32 is always safe here.

---
Commit ac0004e71 authored by Yong Tang
Committed by Vijay Vasudevan:
Add int64 shape support on GPU for stateless random ops. (#13908)

* Add int64 shape support on GPU for stateless random ops.

This fix adds int64 shape support on GPU for stateless random ops
`StatelessRandomUniform`, `StatelessRandomNormal`, `StatelessTruncatedNormal`.

The int64 shape for stateless random ops is already supported on CPU
with int32/int64 processed properly through `MakeShape`.

However, on GPU a type constraint `.TypeConstraint("T")`
has been improperly added. Such a type constraint actually prevents
an int64 shape type to run on GPU. (As a comparision, no type constraint
on CPU).

This fix removes the type constraint and allows int64 shape to be run on GPU.

This fix also adds test cases for int64 shape support on stateless random ops.

Signed-off-by: Yong Tang 

* Add test cases for int64 shape support for stateless random ops.

Signed-off-by: Yong Tang 

* Add int32 to shape types tested.

---
Commit 0d437c3be authored by Yong Tang
Committed by Vijay Vasudevan:
Add int64 padding support for MirrorPad (#13907)

* Add int64 padding support for MirrorPad

This fix adds int64 padding support for `MirrorPad`.
In the `array_ops.cc` the `MirrorPad`/`MirrorPadGrad`
has been specified as supporting int64 padding. The related
kernels does not have the int64 padding registered though.
This fix adds the int64 padding support. This fix also adds
additional test cases for coverage.

Signed-off-by: Yong Tang 

* Update template for CPU and GPU support of int64 paddings.

Signed-off-by: Yong Tang 

* Add int64 padding support for MirrorPad

Signed-off-by: Yong Tang 

* Put eigen header first like before, just in case.

---
Commit 690003cc0 authored by Yong Tang
Committed by Vijay Vasudevan:
Add `int64` type `multiples` support for `tf.tile` (#13884)

* Add `int64` type `multiples` support for `tf.tile`

In the doc of `tf.tile` (tf.tile.__doc__) both `int32`
and `int64` are supported for `multiples`. However, the kernel
for `int64` is not registered yet.

This fix adds the support of `int64` `multiples` so that the
behavior matches the description of the docs.

Signed-off-by: Yong Tang 

* Update functors for int64 multiples support in `tf.tile`

Signed-off-by: Yong Tang 

* Update test cases for int64 of multiples in `tf.tile`

Signed-off-by: Yong Tang 

* Add GPU and non GPU tests

Signed-off-by: Yong Tang 

* format with clang-format -i

Signed-off-by: Yong Tang 

* Move Tmultiples after T (as it is  auxilliary)

And use `use_gpu=True`

Signed-off-by: Yong Tang 

---
Commit fd8d517b9 authored by Yunxing Dai
Committed by TensorFlower Gardener:
Add tests for convolution 1D
RELNOTES: n/a

PiperOrigin-RevId: 173060283

---
Commit 40c475b48 authored by formath
Committed by Vijay Vasudevan:
add segment_reduction_ops to tf_op_files (#13901)

---
Commit bfa4ec194 authored by Tayo Oguntebi<10927929+tayo@users.noreply.github.com>
Committed by Vijay Vasudevan:
Update node_def.proto comments (#13874)

The device field had outdated comments.

Note: We could consider adding tpu as an example here, e.g. "gpu" | "cpu" | "tpu".  Thoughts?
---
Commit c9cb5a58d authored by formath
Committed by Vijay Vasudevan:
protobuf lib path bug fix for benckmark on osx (#13878)

---
Commit 1c1dad105 authored by Yong Tang
Committed by Vijay Vasudevan:
Add int64 axis support for reduction ops. (#13891)

* Add int64 axis support for reduction ops.

This fix is a follow up to PR 13863. In PR 13863 the
program crash is fixed if int64 axis is passed to reduction ops,
e.g. reduce_sum, reduce_max, etc. However, 13863 does not
process the case of int64 support, it merely fixes the crash.

This fix adds the support for int64 axis of reduction ops.

Signed-off-by: Yong Tang 

* Add int64 axis support for mean, prod, sum

Signed-off-by: Yong Tang 

* Add int64 axis support for min and max.

Signed-off-by: Yong Tang 

* Add int64 axis support for reduce_all and reduce_any

Signed-off-by: Yong Tang 

* Add test cases for int64 axis support of reduce_any and reduce_all

Signed-off-by: Yong Tang 

---
Commit 17096081e authored by Yong Tang
Committed by Vijay Vasudevan:
Improve resize_bicubic performance by reorganizing loops (#13840)

* Improve resize_bicubic performance by reorganizing loops

This fix tries to address the issue raised in 13693 where
performance of `resize_bicubic` is not on par with opencv.

This fix rearranges the loops so that it is the same for
num_channel=40 and num_channel=3:

Pre-fix:
```
CHANNEL=40
opencv: 145.08ms
tf: 314.26ms

CHANNEL=3
opencv: 11.95ms
tf: 8.95ms
```

Post-fix:
```
CHANNEL=40
opencv: 144.25ms
tf: 214.55ms

CHANNEL=3
opencv: 11.78ms
tf: 14.07ms
```

This fix fixes 13693.

Signed-off-by: Yong Tang 

* Keep special handling of `num_channels=3` for `resize_bicubic`

This commit keeps special handling of `num_channels=3` for
`resize_bicubic`:
Without special handling:
```
opencv: 11.78ms
tf: 14.07ms
```
With special handling:
```
opencv: 11.74ms
tf: 9.46ms
```

Signed-off-by: Yong Tang 

* Expand Benchmark test for resize_bicubic

Signed-off-by: Yong Tang 

* Update from review feedback.

Signed-off-by: Yong Tang 

---
Commit b927df57f authored by Yong Tang
Committed by Vijay Vasudevan:
Update protobuf.cmake to b04e5cba356212e4e8c66c61bbe0c3a20537c5b9 (#13893)

This fix tries to address the issue raised in 8187 where
protobuf.cmake used different version as bazel.

The reason for discrepancy was due to the fact that a customerized
protobuf was needed with Windows patch. Since the patch has been
merged in (https://github.com/google/protobuf/pull/2203),
it makes sense to update protobuf.cmake so that the same version
of cmake is used.

This fix fixes 8187.

Signed-off-by: Yong Tang 
---
Commit d1183ca6a authored by Vijay Vasudevan
Committed by GitHub:
Give each variable a unique name in accumulate_n_v2_eager_test. (#13886)

---
Commit a69945810 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Update pin for bazel-toolchains to latest version

PiperOrigin-RevId: 173002530

---
Commit 9d55c249c authored by Yong Tang
Committed by Vijay Vasudevan:
Fix doc in TF_CALL_ when invoked in mobile platform (#13881)

* Fix doc in TF_CALL_ when defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__)

This is a small doc fix that includes bool as part of the types
that is supported in mobile (IS_MOBILE_PLATFORM && !__ANDROID_TYPES_FULL__),
as bool is clearly invoked in the following define.

Signed-off-by: Yong Tang 

* Also add bool to android full version.

Signed-off-by: Yong Tang 

---
Commit ba49d8583 authored by Bjarke Hammersholt Roune
Committed by TensorFlower Gardener:
Slight change to reduce_test to avoid generating inf, which was triggering an inf detector unnecessarily.

PiperOrigin-RevId: 172965466

---
Commit 93e8f3c67 authored by Anna R
Committed by TensorFlower Gardener:
Adding Python ApiDef overrides.

PiperOrigin-RevId: 172960496

---
Commit 0d6a2e353 authored by Anna R
Committed by TensorFlower Gardener:
Internal change.

PiperOrigin-RevId: 172960439

---
Commit 62df65c72 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Add dtype argument to Mean and Accuracy object-oriented metrics.

PiperOrigin-RevId: 172957714

---
Commit d7409d32b authored by Simone Cirillo
Committed by Vijay Vasudevan:
Fix import of spatial_softmax from tensorflow.contrib.layers (#13833)

---
Commit df8bce63d authored by Yong Tang
Committed by Vijay Vasudevan:
Fix crash when `int64` axis is passed to `tf.reduce_sum` (#13863)

* Fix crash when `int64` axis is passed to `tf.reduce_sum`

This fix tries to fix the crash triggered by `int64` axis passed
to `tf.reduce_sum`:
```
ubuntu@ubuntu:~/tensorflow2$ (cd && python)
Python 2.7.12 (default, Nov 19 2016, 06:48:10)
[GCC 5.4.0 20160609] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import tensorflow as tf
>>> v = tf.reduce_sum([1,2,3], tf.constant(0, tf.int64))
2017-10-20 15:55:06.993430: F tensorflow/core/framework/tensor.cc:601] Check failed: dtype() == expected_dtype (9 vs. 3)
ubuntu@ubuntu:~/tensorflow2$
```

The issue is caused by the fact that shape inference in `common_shape_fns.cc`
only assumes int32 without proper handling of diffent types. In `math_ops.cc`
both int32 and int64 are mentioned.

NOTE that this fix does not address the issue that int64 is not supported.
To allow int64 axis it is more than adding a template in `ReductionOp` as the type
of the axis seems to be decided by some other ways in Eigen.

This fix merely fixed the crash so that an error message will return without
exit from the python program "No OpKernel was registered to support Op 'Sum' with these attrs".

Still, I think its worth to at least allow the program to continue in case of unsupported kernel.

Signed-off-by: Yong Tang 

* Update implementation with a template helper function.

Signed-off-by: Yong Tang 

---
Commit 29c7b4658 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Adding the Stanford Tensorflow class to community resources.

PiperOrigin-RevId: 172956049

---
Commit f758b24a8 authored by Alexandre Passos
Committed by Vijay Vasudevan:
Variable name for the eager test (#13873)

---
Commit a5fe66b15 authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Removed some unnecessary broadcasts in binary ops where only one input needs
broadcasting (which is a fairly common case, even in the fallback path).

PiperOrigin-RevId: 172950493

---
Commit c77090a0a authored by Yong Tang
Committed by Vijay Vasudevan:
Fix issues where int64 crops could not be passed to batch_to_space. (#13862)

* Fix issues where int64 crops could not be passed to batch_to_space.

This fix tries to address the issue where int64 `crops` could
not be passed to `batch_to_space` even though both int32 and
int64 are specified as supported in the docs (tf.batch_to_space.__doc__)

The reason is that BatchToSpace kernel puts a constraint of int32 to crops
data types.

This fix removed the constraint so that int64 `crops` could be supported.

NOTE: Just removing the constraint should work and it is not necessary
to add specification to the kernel class template, as `SubtleMustCopyFlat`
called in the class already correctly handled both int32 and int64 cases.
Besides, other data types (e.g., float or double) will not be passed to the
kernel as they are guarded by the specification in `array_ops.cc`.

Signed-off-by: Yong Tang 

* Also remove int64/int32 type constraints for SpaceToBatch kernels

Signed-off-by: Yong Tang 

* Add test cases for int64 crops of batch_to_space and space_to_batch

Signed-off-by: Yong Tang 

* Fix test failures.

Signed-off-by: Yong Tang 

---
Commit 494837936 authored by Joshua V. Dillon
Committed by TensorFlower Gardener:
Make `tf.contrib.distributions` quadrature family accept a `Tensor` for
`quadrature_grid_and_probs` argument.

PiperOrigin-RevId: 172950094

---
Commit 9c825d32c authored by Jinze Bai
Committed by Vijay Vasudevan:
Merge two GPU kernel launching to one in DiagOp. (#13859)

---
Commit c0ca50a47 authored by Yan Facai (???)
Committed by Vijay Vasudevan:
ENH: add Relu6GradGrad (#13268)

* ENH: add Relu6GradGrad

* TST: add test case

* CLN: import nn_grad

* TST: add init value

---
Commit 8ff33271e authored by Justin Lebar
Committed by TensorFlower Gardener:
Dump the computation's SessionModule as part of the tf_compile rule.

PiperOrigin-RevId: 172946149

---
Commit ebcae4a5e authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
Add streaming_precision_recall_at_equal_thresholds

This helper method computes streaming tp, fp, tn, fp, precision, and recall for the user in a way that exhibits O(T + N) time and space complexity (instead of O(T * N)), where T is the number of thresholds and N is the size of the predictions tensor.

Thanks to Frank Chu for the efficient algorithm!

PiperOrigin-RevId: 172946073

---
Commit ccfd9c1e5 authored by Sanjoy Das
Committed by TensorFlower Gardener:
Log Hlo IR during AOT compilation

PiperOrigin-RevId: 172944165

---
Commit 985031a10 authored by Alexandre Passos
Committed by TensorFlower Gardener:
Allows tfe.enable_eager_execution(device_policy=tfe.DEVICE_POLICY_WARN).

PiperOrigin-RevId: 172943398

---
Commit 703182d85 authored by Mingxing Tan
Committed by TensorFlower Gardener:
Add performance guide for fused decode_and_crop_jpeg optimization.

PiperOrigin-RevId: 172943116

---
Commit 66b1f4383 authored by Francois Chollet
Committed by TensorFlower Gardener:
Make Network compatible with eager mode. Currently it only allows to instantiate a Network in eager mode using the regular Keras API, and call it on eager tensors.

PiperOrigin-RevId: 172942569

---
Commit 41df2cec2 authored by ashankar
Committed by TensorFlower Gardener:
Testing pending CL: 172939383

---
Commit 37fd95179 authored by Alexandre Passos
Committed by TensorFlower Gardener:
Simplifies capturing code in graph_callable to use recent function improvements.

PiperOrigin-RevId: 172937003

---
Commit d1e7382af authored by A. Unique TensorFlower
Committed by TensorFlower Gardener:
BEGIN_PUBLIC
Automated g4 rollback of changelist 172924803

PiperOrigin-RevId: 173347587
---
 .gitignore                                    |   5 +
 README.md                                     |   9 +-
 RELEASE.md                                    |  81 ++++-
 configure.py                                  |   5 +-
 tensorflow/BUILD                              |   1 +
 tensorflow/c/c_api.h                          |   2 +-
 tensorflow/cc/gradients/math_grad.cc          |  32 +-
 tensorflow/cc/gradients/math_grad_test.cc     |  58 +++-
 tensorflow/compiler/jit/BUILD                 |   1 +
 tensorflow/compiler/plugin/BUILD              |  56 ++++
 tensorflow/compiler/plugin/README.md          |  16 +
 .../xla/service/hlo_computation_test.cc       |   2 +-
 tensorflow/compiler/xla/service/inliner.cc    |   6 +-
 .../compiler/xla/service/inliner_test.cc      |  39 +++
 .../contrib/all_reduce/python/all_reduce.py   |   2 +-
 tensorflow/contrib/boosted_trees/README.md    |   2 +-
 .../boosted_trees/examples/binary_mnist.py    |   2 +-
 .../contrib/boosted_trees/examples/mnist.py   |   2 +-
 tensorflow/contrib/cmake/external/cub.cmake   |   4 +-
 .../contrib/cmake/external/protobuf.cmake     |   4 +-
 .../contrib/cmake/tf_core_kernels.cmake       |   4 +
 tensorflow/contrib/cmake/tf_tests.cmake       |  90 +++---
 .../python/kernel_tests/cudnn_rnn_test.py     |   6 +-
 .../contrib/data/python/kernel_tests/BUILD    |   6 +
 tensorflow/contrib/framework/BUILD            |  29 +-
 .../framework/python/ops/accumulate_n_v2.py   | 111 +++++++
 .../python/ops/accumulate_n_v2_eager_test.py  |  85 +++++
 .../python/ops/accumulate_n_v2_test.py        | 123 ++++++++
 tensorflow/contrib/image/__init__.py          |   4 +
 .../python/kernel_tests/image_ops_test.py     |  33 +-
 .../contrib/image/python/ops/image_ops.py     | 294 +++++++++++------
 .../contrib/kfac/python/ops/loss_functions.py |   6 +-
 .../contrib/kfac/python/ops/op_queue.py       |   2 +-
 tensorflow/contrib/layers/__init__.py         |   1 +
 .../learn/python/learn/learn_runner.py        |   2 +-
 .../contrib/losses/python/losses/loss_ops.py  |  17 +-
 tensorflow/contrib/makefile/Makefile          |   4 +
 .../contrib/makefile/download_dependencies.sh |   2 +-
 tensorflow/contrib/makefile/tf_op_files.txt   |   1 +
 .../meta_graph_transform.py                   |   2 +-
 .../metrics/python/ops/metric_ops_test.py     |  54 +++-
 .../contrib/mpi_collectives/__init__.py       |   2 +-
 tensorflow/contrib/nn/__init__.py             |   2 +
 .../python/util/receptive_field.py            | 134 +++++++-
 .../python/util/receptive_field_test.py       |  56 ++++
 .../kernel_tests/stateless_random_ops_test.py |  16 +
 tensorflow/core/BUILD                         |  34 +-
 .../common_runtime/accumulate_n_optimizer.cc  | 191 ++++++++++++
 .../core/common_runtime/mkl_cpu_allocator.h   |  61 +++-
 .../common_runtime/mkl_cpu_allocator_test.cc  |  53 ++++
 tensorflow/core/framework/common_shape_fns.cc |  58 ++--
 tensorflow/core/framework/node_def.proto      |   2 +-
 tensorflow/core/framework/register_types.h    |   4 +-
 tensorflow/core/framework/rendezvous.cc       |   2 +-
 tensorflow/core/graph/graph.h                 |   2 +-
 tensorflow/core/graph/mkl_layout_pass.cc      |   2 +-
 tensorflow/core/graph/mkl_layout_pass_test.cc |   2 +-
 tensorflow/core/graph/testlib.cc              |  18 ++
 tensorflow/core/graph/testlib.h               |   6 +
 .../core/grappler/optimizers/model_pruner.cc  |   2 +-
 tensorflow/core/kernels/BUILD                 |  55 ++++
 tensorflow/core/kernels/batchtospace_op.cc    |  50 ++-
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc  |   2 +-
 .../core/kernels/crop_and_resize_op_test.cc   |   6 +-
 tensorflow/core/kernels/dataset.h             |   2 +-
 tensorflow/core/kernels/diag_op.cc            | 295 +++++++++++-------
 tensorflow/core/kernels/diag_op.h             |  43 +++
 tensorflow/core/kernels/diag_op_gpu.cu.cc     | 139 +++++++++
 tensorflow/core/kernels/diag_op_test.cc       |  54 ++++
 tensorflow/core/kernels/histogram_op.cc       | 147 +++++++++
 tensorflow/core/kernels/histogram_op.h        |  38 +++
 .../core/kernels/histogram_op_gpu.cu.cc       | 125 ++++++++
 tensorflow/core/kernels/listdiff_op.cc        |  16 +-
 tensorflow/core/kernels/map_stage_op.cc       |  12 +-
 tensorflow/core/kernels/mirror_pad_op.cc      | 200 +++++++-----
 tensorflow/core/kernels/mirror_pad_op.h       |  13 +-
 .../core/kernels/mirror_pad_op_cpu_impl.h     |  12 +-
 .../core/kernels/mirror_pad_op_gpu.cu.cc      |  32 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |   6 +-
 tensorflow/core/kernels/nth_element_op.cc     | 139 +++++++++
 tensorflow/core/kernels/nth_element_op.h      |  39 +++
 tensorflow/core/kernels/pad_op.cc             | 144 +++++++--
 tensorflow/core/kernels/pad_op.h              |  10 +-
 tensorflow/core/kernels/pad_op_gpu.cu.cc      |  20 +-
 tensorflow/core/kernels/reduction_ops_all.cc  |  16 +-
 tensorflow/core/kernels/reduction_ops_any.cc  |  16 +-
 .../core/kernels/reduction_ops_common.cc      |  22 +-
 .../core/kernels/reduction_ops_common.h       |  27 +-
 tensorflow/core/kernels/reduction_ops_max.cc  |  90 ++++--
 tensorflow/core/kernels/reduction_ops_mean.cc |  68 ++--
 tensorflow/core/kernels/reduction_ops_min.cc  |  90 ++++--
 tensorflow/core/kernels/reduction_ops_prod.cc |  68 ++--
 tensorflow/core/kernels/reduction_ops_sum.cc  |  90 ++++--
 tensorflow/core/kernels/resize_bicubic_op.cc  |  85 ++---
 .../core/kernels/resize_bicubic_op_test.cc    |  20 +-
 .../core/kernels/reverse_sequence_op.cc       |   3 +
 .../kernels/reverse_sequence_op_gpu.cu.cc     |   1 +
 tensorflow/core/kernels/scan_ops.cc           |  98 +++---
 tensorflow/core/kernels/sequence_ops.cc       |  48 +--
 tensorflow/core/kernels/sequence_ops_test.cc  | 148 +++++++++
 tensorflow/core/kernels/spacetobatch_op.cc    |  50 ++-
 tensorflow/core/kernels/sparse_matmul_op.h    |   6 +-
 tensorflow/core/kernels/stage_op.cc           |  14 +-
 .../core/kernels/stateless_random_ops.cc      |   3 -
 tensorflow/core/kernels/tile_functor.h        |  39 ++-
 tensorflow/core/kernels/tile_functor_cpu.cc   |  12 +-
 .../core/kernels/tile_functor_gpu.cu.cc       |  12 +-
 tensorflow/core/kernels/tile_ops.cc           | 249 ++++++---------
 tensorflow/core/kernels/transpose_op.cc       | 134 ++++----
 tensorflow/core/ops/array_ops.cc              |  44 ++-
 tensorflow/core/ops/array_ops_test.cc         |  13 +-
 tensorflow/core/ops/image_ops.cc              |  43 ++-
 tensorflow/core/ops/image_ops_test.cc         |   6 +-
 tensorflow/core/ops/math_ops.cc               |  72 ++++-
 tensorflow/core/ops/nn_ops.cc                 |  50 +++
 tensorflow/core/ops/nn_ops_test.cc            |  24 ++
 tensorflow/core/platform/s3/s3_crypto.cc      |   2 +-
 tensorflow/core/profiler/README.md            |   2 +-
 tensorflow/core/profiler/g3doc/options.md     |   2 +-
 tensorflow/core/public/version.h              |   4 +-
 .../api_guides/python/reading_data.md         |   2 +-
 tensorflow/docs_src/get_started/estimator.md  |  20 +-
 tensorflow/docs_src/install/install_c.md      |   2 +-
 tensorflow/docs_src/install/install_go.md     |   2 +-
 tensorflow/docs_src/install/install_java.md   |  18 +-
 tensorflow/docs_src/install/install_linux.md  |  38 ++-
 tensorflow/docs_src/install/install_mac.md    |  12 +-
 .../docs_src/install/install_sources.md       |  25 +-
 .../docs_src/install/install_windows.md       |   2 +-
 .../docs_src/performance/performance_guide.md |   2 +-
 .../performance/performance_models.md         |   2 +-
 .../docs_src/programmers_guide/datasets.md    |   2 +-
 .../docs_src/programmers_guide/graphs.md      |   4 +-
 .../docs_src/programmers_guide/saved_model.md |  33 ++
 tensorflow/docs_src/tutorials/wide.md         |   3 +-
 .../get_started/regression/imports85.py       |   2 +-
 .../linear_regression_categorical.py          |   2 +-
 tensorflow/examples/learn/resnet.py           |   4 +-
 .../tutorials/word2vec/word2vec_basic.py      |   6 +-
 tensorflow/java/BUILD                         |  39 +++
 .../processor/OperatorProcessor.java          | 164 ++++++++++
 .../javax.annotation.processing.Processor     |   1 +
 .../tensorflow/op/annotation/Operator.java    |   2 +-
 .../processor/OperatorProcessorTest.java      |  51 +++
 .../processor/operator/bad/BasicBad.java      |  22 ++
 .../processor/operator/good/BasicGood.java    |  21 ++
 tensorflow/python/BUILD                       |  13 +
 tensorflow/python/debug/cli/tensor_format.py  |   2 +-
 tensorflow/python/estimator/training.py       |   6 +-
 tensorflow/python/kernel_tests/BUILD          |  15 +
 .../kernel_tests/batchtospace_op_test.py      |  36 ++-
 .../python/kernel_tests/diag_op_test.py       |  64 +++-
 .../python/kernel_tests/listdiff_op_test.py   |  20 +-
 .../python/kernel_tests/metrics_test.py       |  51 ++-
 .../kernel_tests/nth_element_op_test.py       | 174 +++++++++++
 tensorflow/python/kernel_tests/pad_op_test.py |  28 ++
 .../python/kernel_tests/reduction_ops_test.py |  52 +++
 .../python/kernel_tests/scan_ops_test.py      |  18 ++
 .../python/kernel_tests/shape_ops_test.py     |  18 +-
 .../python/kernel_tests/slice_op_test.py      |  11 +
 .../python/kernel_tests/transpose_op_test.py  |  13 +
 tensorflow/python/ops/hidden_ops.txt          |   3 +
 tensorflow/python/ops/histogram_ops.py        |  31 +-
 tensorflow/python/ops/histogram_ops_test.py   |   8 +-
 tensorflow/python/ops/image_ops_test.py       |  29 +-
 tensorflow/python/ops/losses/losses_impl.py   |  22 +-
 tensorflow/python/ops/metrics_impl.py         |  14 +-
 tensorflow/python/ops/nn_grad.py              |  36 +++
 tensorflow/python/ops/nn_grad_test.py         |  48 +++
 tensorflow/python/ops/nn_ops.py               |  28 ++
 tensorflow/python/platform/self_check.py      |   8 +-
 .../tools/api/golden/tensorflow.losses.pbtxt  |   2 +-
 tensorflow/tools/ci_build/Dockerfile.pi       |   3 +
 .../tools/ci_build/Dockerfile.pi-python3      |  23 ++
 tensorflow/tools/ci_build/README.md           | 143 +++------
 .../tools/ci_build/builds/android_full.sh     |   4 +-
 .../tools/ci_build/builds/libtensorflow.sh    |  45 ++-
 .../tools/ci_build/install/install_golang.sh  |   2 +-
 .../install/install_pi_python3_toolchain.sh   |  29 ++
 .../ci_build/install/install_pi_toolchain.sh  |   2 +-
 .../tools/ci_build/linux/cpu/run_mkl.sh       |  36 +++
 tensorflow/tools/docker/Dockerfile            |   2 +-
 tensorflow/tools/docker/Dockerfile.devel      |   4 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |  15 +-
 .../docker/Dockerfile.devel-gpu-cuda9-cudnn7  |  33 +-
 tensorflow/tools/docker/Dockerfile.gpu        |   2 +-
 tensorflow/tools/docker/README.md             |   1 +
 tensorflow/tools/pip_package/setup.py         |   5 +-
 tensorflow/workspace.bzl                      |  44 ++-
 third_party/aws.BUILD                         |   1 +
 .../CXX11/src/FixedPoint/PacketMathAVX2.h     |  51 ++-
 third_party/toolchains/cpus/arm/CROSSTOOL.tpl |   2 +-
 .../cpus/arm/arm_compiler_configure.bzl       |  11 +
 193 files changed, 5403 insertions(+), 1408 deletions(-)
 create mode 100644 tensorflow/compiler/plugin/BUILD
 create mode 100644 tensorflow/compiler/plugin/README.md
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
 create mode 100644 tensorflow/core/common_runtime/accumulate_n_optimizer.cc
 create mode 100644 tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
 create mode 100644 tensorflow/core/kernels/diag_op.h
 create mode 100644 tensorflow/core/kernels/diag_op_gpu.cu.cc
 create mode 100644 tensorflow/core/kernels/diag_op_test.cc
 create mode 100644 tensorflow/core/kernels/histogram_op.cc
 create mode 100644 tensorflow/core/kernels/histogram_op.h
 create mode 100644 tensorflow/core/kernels/histogram_op_gpu.cu.cc
 create mode 100644 tensorflow/core/kernels/nth_element_op.cc
 create mode 100644 tensorflow/core/kernels/nth_element_op.h
 create mode 100644 tensorflow/core/kernels/sequence_ops_test.cc
 create mode 100644 tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
 create mode 100644 tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
 create mode 100644 tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
 create mode 100644 tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
 create mode 100644 tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java
 create mode 100644 tensorflow/python/kernel_tests/nth_element_op_test.py
 create mode 100644 tensorflow/python/ops/nn_grad_test.py
 create mode 100644 tensorflow/tools/ci_build/Dockerfile.pi-python3
 create mode 100755 tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh
 create mode 100755 tensorflow/tools/ci_build/linux/cpu/run_mkl.sh

diff --git a/.gitignore b/.gitignore
index 09734fe497..9ae0d9c96f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,8 @@ cmake_build/
 .idea/**
 /build/
 /tensorflow/core/util/version_info.cc
+/tensorflow/python/framework/fast_tensor_util.cpp
+Pods
+Podfile.lock
+*.pbxproj
+*.xcworkspacedata
diff --git a/README.md b/README.md
index 6339c57c95..24bbb6cec1 100644
--- a/README.md
+++ b/README.md
@@ -38,10 +38,11 @@ People who are a little more adventurous can also try our nightly binaries:
 
 **Nightly pip packages**
 * We are pleased to announce that TensorFlow now offers nightly pip packages
-under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) project on pypi.
-Simply run `pip install tf-nightly` in a clean environment to install the nightly
-tensorflow  build. We currently only support CPU packages on Linux, Mac, and Windows.
-GPU packages on all platforms will arrive soon!
+under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) and
+[tf-nightly-gpu](https://pypi.python.org/pypi/tf-nightly-gpu) project on pypi.
+Simply run `pip install tf-nightly` or `pip install tf-nightly-gpu` in a clean
+environment to install the nightly TensorFlow build. We support CPU and GPU
+packages on Linux, Mac, and Windows.
 
 
 **Individual whl files**
diff --git a/RELEASE.md b/RELEASE.md
index 2c6535c15d..4a33bce8b2 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,20 +1,51 @@
 # Release 1.4.0
 
 ## Major Features And Improvements
+* `tf.keras` is now part of the core TensorFlow API.
 * [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of
   the core TensorFlow API.
   * The API is now subject to backwards compatibility guarantees.
   * For a guide to migrating from the `tf.contrib.data` API, see the
-    [README] (https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
+    [README](https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
   * Major new features include `Dataset.from_generator()` (for building an input
     pipeline from a Python generator), and the `Dataset.apply()` method for
     applying custom transformation functions.
   * Several custom transformation functions have been added, including
     `tf.contrib.data.batch_and_drop_remainder()` and
     `tf.contrib.data.sloppy_interleave()`.
+* Add `train_and_evaluate` for simple distributed `Estimator` training.
+* Add `tf.spectral.dct` for computing the DCT-II.
+* Add Mel-Frequency Cepstral Coefficient support to `tf.contrib.signal`
+  (with GPU and gradient support).
+* Add a self-check on `import tensorflow` for Windows DLL issues.
+* Add NCHW support to `tf.depth_to_space` on GPU.
+* SinhArcsinh (scalar) distribution added to `contrib.distributions`.
+* Make `GANEstimator` opensource.
+* `Estimator.export_savedmodel()` now includes all valid serving signatures
+  that can be constructed from the Serving Input Receiver and all available
+  ExportOutputs. For instance, a classifier may provide regression- and
+  prediction-flavored outputs, in addition to the classification-flavored one.
+  Building signatures from these allows TF Serving to honor requests using the
+  different APIs (Classify, Regress, and Predict). Furthermore,
+  `serving_input_receiver_fn()` may now specify alternative subsets of nodes
+  that may act as inputs. This allows, for instance, producing a prediction
+  signature for a classifier that accepts raw `Tensors` instead of a serialized
+  `tf.Example`.
+* Add `tf.contrib.bayesflow.hmc`.
+* Add `tf.contrib.distributions.MixtureSameFamily`.
+* Make `Dataset.shuffle()` always reshuffles after each iteration by default.
+* Add `tf.contrib.bayesflow.metropolis_hastings`.
+* Add `log_rate` parameter to `tf.contrib.distributions.Poisson`.
+* Extend `tf.contrib.distributions.bijector` API to handle some non-injective
+  transforms.
 * Java:
-  * Generics (e.g., `Tensor`) for improved type-safety (courtesy @andrewcmyers).
+  * Generics (e.g., `Tensor`) for improved type-safety
+    (courtesy @andrewcmyers).
   * Support for multi-dimensional string tensors.
+  * Support loading of custom operations (e.g. many in `tf.contrib`) on Linux
+    and OS X
+* All our prebuilt binaries have been built with CUDA 8 and cuDNN 6.
+  We anticipate releasing TensorFlow 1.5 with CUDA 9 and cuDNN 7.
 
 ## Bug Fixes and Other Changes
 * `tf.nn.rnn_cell.DropoutWrapper` is now more careful about dropping out LSTM
@@ -26,11 +57,57 @@
 * Removed `tf.contrib.training.python_input`.  The same behavior, in a more
   flexible and reproducible package, is available via the new
   `tf.contrib.data.Dataset.from_generator` method!
+* Fix `tf.contrib.distributions.Affine` incorrectly computing log-det-jacobian.
+* Fix `tf.random_gamma` incorrectly handling non-batch, scalar draws.
+* Resolved a race condition in TensorForest TreePredictionsV4Op.
+* Google Cloud Storage file system and Hadoop file system support are now
+  default build options.
+* Custom op libraries must link against libtensorflow_framework.so
+  (installed at `tf.sysconfig.get_lib()`).
 
 ## Breaking Changes to the API
 * The signature of the `tf.contrib.data.rejection_resample()` function has been
   changed. It now returns a function that can be used as an argument to
   `Dataset.apply()`.
+* Remove `tf.contrib.data.Iterator.from_dataset()` method. Use
+  `Dataset.make_initializable_iterator()` instead.
+* Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`.
+* Reorder some TFGAN loss functions in a non-backwards compatible way.
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Google, as well as:
+
+4d55397500, Abdullah Alrasheed, abenmao, Adam Salvail, Aditya Dhulipala, Ag Ramesh,
+Akimasa Kimura, Alan Du, Alan Yee, Alexander, Amit Kushwaha, Amy, Andrei Costinescu,
+Andrei Nigmatulin, Andrew Erlichson, Andrew Myers, Andrew Stepanov, Androbin, AngryPowman,
+Anish Shah, Anton Daitche, Artsiom Chapialiou, asdf2014, Aseem Raj Baranwal, Ash Hall,
+Bart Kiers, Batchu Venkat Vishal, ben, Ben Barsdell, Bill Piel, Carl Thomé, Catalin Voss,
+Changming Sun, Chengzhi Chen, Chi Zeng, Chris Antaki, Chris Donahue, Chris Oelmueller,
+Chris Tava, Clayne Robison, Codrut, Courtial Florian, Dalmo Cirne, Dan J, Darren Garvey,
+David Kristoffersson, David Norman, David RöThlisberger, DavidNorman, Dhruv, DimanNe,
+Dorokhov, Duncan Mac-Vicar P, EdwardDixon, EMCP, error.d, FAIJUL, Fan Xia,
+Francois Xavier, Fred Reiss, Freedom" Koan-Sin Tan, Fritz Obermeyer, Gao, Xiang,
+Guenther Schmuelling, Guo Yejun (郭叶军), Hans Gaiser, HectorSVC, Hyungsuk Yoon,
+James Pruegsanusak, Jay Young, Jean Wanka, Jeff Carpenter, Jeremy Rutman, Jeroen BéDorf,
+Jett Jones, Jimmy Jia, jinghuangintel, jinze1994, JKurland, Joel Hestness, joetoth,
+John B Nelson, John Impallomeni, John Lawson, Jonas, Jonathan Dekhtiar, joshkyh, Jun Luan,
+Jun Mei, Kai Sasaki, Karl Lessard, karl@kubx.ca, Kb Sriram, Kenichi Ueno, Kevin Slagle,
+Kongsea, Lakshay Garg, lhlmgr, Lin Min, liu.guangcong, Loki Der Quaeler, Louie Helm,
+lucasmoura, Luke Iwanski, Lyndon White, Mahmoud Abuzaina, Marcel Puyat, Mark Aaron Shirley,
+Michele Colombo, MtDersvan, Namrata-Ibm, Nathan Luehr, Naurril, Nayana Thorat, Nicolas Lopez,
+Niranjan Hasabnis, Nolan Liu, Nouce, Oliver Hennigh, osdamv, Patrik Erdes,
+Patryk Chrabaszcz, Pavel Christof, Penghao Cen, postBG, Qingqing Cao, Qingying Chen, qjivy,
+Raphael, Rasmi, raymondxyang, Renze Yu, resec, Roffel, Ruben Vereecken, Ryohei Kuroki,
+sandipmgiri, Santiago Castro, Scott Kirkland, Sean Vig, Sebastian Raschka, Sebastian Weiss,
+Sergey Kolesnikov, Sergii Khomenko, Shahid, Shivam Kotwalia, Stuart Berg, Sumit Gouthaman,
+superzerg, Sven Mayer, tetris, Ti Zhou, Tiago Freitas Pereira, Tian Jin, Tomoaki Oiki,
+Vaibhav Sood, vfdev, Vivek Rane, Vladimir Moskva, wangqr, Weber Xie, Will Frey,
+Yan Facai (颜发才), yanivbl6, Yaroslav Bulatov, Yixing Lao, Yong Tang, youkaichao,
+Yuan (Terry) Tang, Yue Zhang, Yuxin Wu, Ziming Dong, ZxYuan, 黄璞
+
+We are also grateful to all who filed issues or helped resolve them, asked and
+answered questions, and were part of inspiring discussions.
 
 # Release 1.3.0
 
diff --git a/configure.py b/configure.py
index ea3f598f3d..425eae676c 100644
--- a/configure.py
+++ b/configure.py
@@ -989,6 +989,7 @@ def main():
   run_gen_git_source(environ_cp)
 
   if is_windows():
+    environ_cp['TF_NEED_S3'] = '0'
     environ_cp['TF_NEED_GCP'] = '0'
     environ_cp['TF_NEED_HDFS'] = '0'
     environ_cp['TF_NEED_JEMALLOC'] = '0'
@@ -1001,9 +1002,9 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc',
                 'with_jemalloc', True)
   set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform',
-                'with_gcp_support', False, 'gcp')
+                'with_gcp_support', True, 'gcp')
   set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
-                'with_hdfs_support', False, 'hdfs')
+                'with_hdfs_support', True, 'hdfs')
   set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
                 'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 673e433a8a..20f02ad50a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -323,6 +323,7 @@ filegroup(
         "//tensorflow/compiler/jit/kernels:all_files",
         "//tensorflow/compiler/jit/legacy_flags:all_files",
         "//tensorflow/compiler/jit/ops:all_files",
+        "//tensorflow/compiler/plugin:all_files",
         "//tensorflow/compiler/tests:all_files",
         "//tensorflow/compiler/tf2xla:all_files",
         "//tensorflow/compiler/tf2xla/cc:all_files",
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 0c6bb53d01..1e8bfdc7b0 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1153,7 +1153,7 @@ TF_CAPI_EXPORT extern TF_Function* TF_FunctionImportFunctionDef(
     const void* proto, size_t proto_len, TF_Status* status);
 
 // Sets function attribute named `attr_name` to value stored in `proto`.
-// If this attribute is already set to another value, it is overriden.
+// If this attribute is already set to another value, it is overridden.
 // `proto` should point to a sequence of bytes of length `proto_len`
 // representing a binary serialization of an AttrValue protocol
 // buffer.
diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc
index 2417bf18a9..d7446b9560 100644
--- a/tensorflow/cc/gradients/math_grad.cc
+++ b/tensorflow/cc/gradients/math_grad.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#define _USE_MATH_DEFINES
+#include 
+
 #include "tensorflow/cc/ops/array_ops_internal.h"
 #include "tensorflow/cc/ops/math_ops_internal.h"
 #include "tensorflow/cc/ops/standard_ops.h"
@@ -200,8 +203,8 @@ Status TanhGrad(const Scope& scope, const Operation& op,
   // evaluated.
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto y = ConjugateHelper(grad_scope, op.output(0));
-  grad_outputs->push_back(internal::TanhGrad(scope, y, grad));
-  return scope.status();
+  grad_outputs->push_back(internal::TanhGrad(grad_scope, y, grad));
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Tanh", TanhGrad);
 
@@ -256,8 +259,8 @@ Status SigmoidGrad(const Scope& scope, const Operation& op,
   // evaluated.
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto y = ConjugateHelper(grad_scope, op.output(0));
-  grad_outputs->push_back(internal::SigmoidGrad(scope, y, grad));
-  return scope.status();
+  grad_outputs->push_back(internal::SigmoidGrad(grad_scope, y, grad));
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Sigmoid", SigmoidGrad);
 
@@ -696,15 +699,32 @@ Status MeanGrad(const Scope& scope, const Operation& op,
 }
 REGISTER_GRADIENT_OP("Mean", MeanGrad);
 
+Status ErfGrad(const Scope& scope, const Operation& op,
+               const std::vector& grad_inputs,
+               std::vector* grad_outputs) {
+  auto grad = grad_inputs[0];
+  auto two_over_root_pi = Cast(scope, Const(scope, 2 / std::sqrt(M_PI)),
+                               grad.type());
+  Scope grad_scope = scope.WithControlDependencies(grad);
+  auto x = ConjugateHelper(grad_scope, op.input(0));
+  // grad * 2/sqrt(pi) * exp(-x**2)
+  auto dx = Mul(grad_scope,
+                Mul(grad_scope, grad, two_over_root_pi),
+                Exp(grad_scope, Neg(grad_scope, Square(grad_scope, x))));
+  grad_outputs->push_back(dx);
+  return grad_scope.status();
+}
+REGISTER_GRADIENT_OP("Erf", ErfGrad);
+
 Status LgammaGrad(const Scope& scope, const Operation& op,
                   const std::vector& grad_inputs,
                   std::vector* grad_outputs) {
   auto grad = grad_inputs[0];
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto x = ConjugateHelper(grad_scope, op.input(0));
-  auto dx = Mul(scope, grad, Digamma(scope, x));
+  auto dx = Mul(grad_scope, grad, Digamma(grad_scope, x));
   grad_outputs->push_back(dx);
-  return scope.status();
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Lgamma", LgammaGrad);
 
diff --git a/tensorflow/cc/gradients/math_grad_test.cc b/tensorflow/cc/gradients/math_grad_test.cc
index a174f223ad..6313f41da5 100644
--- a/tensorflow/cc/gradients/math_grad_test.cc
+++ b/tensorflow/cc/gradients/math_grad_test.cc
@@ -64,7 +64,9 @@ class CWiseUnaryGradTest : public ::testing::Test {
     IMAG,
     CONJ,
     COMPLEX,
-    ANGLE
+    ANGLE,
+    LGAMMA,
+    ERF
   };
 
   template 
@@ -168,6 +170,12 @@ class CWiseUnaryGradTest : public ::testing::Test {
       case ANGLE:
         y = Angle(scope_, x);
         break;
+      case LGAMMA:
+        y = Lgamma(scope_, x);
+        break;
+      case ERF:
+        y = Erf(scope_, x);
+        break;
     }
 
     float max_error;
@@ -503,6 +511,42 @@ TEST_F(CWiseUnaryGradTest, Angle) {
   TestCWiseGrad(ANGLE, x_fn);
 }
 
+TEST_F(CWiseUnaryGradTest, Lgamma) {
+  auto x_fn = [this](const int i) {
+    return RV({-3.5, -2.5, -1.5, 1.0, 2.0, 3.5});
+  };
+  TestCWiseGrad(LGAMMA, x_fn);
+}
+
+TEST_F(CWiseUnaryGradTest, Lgamma_Complex) {
+  auto x_fn = [this](const int i) {
+    return CRV({{-3.5, 0.5}, {-1.5, -0.5}, {1.5, -1.0}, {3.5, 1.0}});
+  };
+  // TODO(kbsriram)
+  // Add test when the lgamma kernel supports complex numbers
+  if (false) {
+    TestCWiseGrad(LGAMMA, x_fn);
+  }
+}
+
+TEST_F(CWiseUnaryGradTest, Erf) {
+  auto x_fn = [this](const int i) {
+    return RV({-1.2, -1.0, -0.5, 0.3, 0.5, 1.3});
+  };
+  TestCWiseGrad(ERF, x_fn);
+}
+
+TEST_F(CWiseUnaryGradTest, Erf_Complex) {
+  auto x_fn = [this](const int i) {
+    return CRV({{-1.2, 0.5}, {-0.5, -0.5}, {0.5, 0.5}, {1.2, -0.5}});
+  };
+  // TODO(kbsriram)
+  // Add test when the erf kernel supports complex numbers
+  if (false) {
+    TestCWiseGrad(ERF, x_fn);
+  }
+}
+
 class MathGradTest : public ::testing::Test {
  protected:
   MathGradTest() : root_(Scope::NewRootScope().WithDevice("/cpu:0")) {}
@@ -821,17 +865,5 @@ TEST_F(NaryGradTest, Minimum) {
   RunTest(x, x_init_value, y, shape);
 }
 
-TEST_F(NaryGradTest, Lgamma) {
-  TensorShape shape({3, 2});
-  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
-  auto y = Lgamma(scope_, x);
-  // Select values to avoid instability when computing finite differences.
-  // Ref: https://en.wikipedia.org/wiki/File:Gamma_plot.svg
-  Tensor x_init_value =
-      test::AsTensor({-3.5f, -2.5f, -1.5f, 1.0f, 2.0f, 3.5f}, {3, 2});
-  RunTest(x, x_init_value, y, shape);
-  // TODO(suharshs): add test case for complex values
-}
-
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index bf63b7e501..bf7d9cf14d 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -33,6 +33,7 @@ cc_library(
     deps = [
         ":xla_cpu_device",
         ":xla_cpu_jit",
+        "//tensorflow/compiler/plugin",
     ] + if_cuda_is_configured([
         ":xla_gpu_device",
         ":xla_gpu_jit",
diff --git a/tensorflow/compiler/plugin/BUILD b/tensorflow/compiler/plugin/BUILD
new file mode 100644
index 0000000000..c1edf2448c
--- /dev/null
+++ b/tensorflow/compiler/plugin/BUILD
@@ -0,0 +1,56 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Configuration file for an XLA plugin.
+
+  please don't check in changes to this file. to prevent changes appearing
+  in git status, use:
+
+  git update-index --assume-unchanged tensorflow/compiler/plugin/BUILD
+
+  To add additional devices to the XLA subsystem, add targets to the
+  dependency list in the 'plugin' target. For instance:
+
+    deps = ["//tensorflow/compiler/plugin/example:plugin_lib"],
+
+  ** Please don't remove this file - it is supporting some 3rd party plugins **
+"""
+
+licenses(["notice"])
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "plugin",
+    deps = [
+        #"//tensorflow/compiler/plugin/example:example_lib",
+    ],
+)
+
+#-----------------------------------------------------------------------------
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/compiler/plugin/README.md b/tensorflow/compiler/plugin/README.md
new file mode 100644
index 0000000000..9dd0d2bdab
--- /dev/null
+++ b/tensorflow/compiler/plugin/README.md
@@ -0,0 +1,16 @@
+3rd party XLA devices
+---------------------
+
+This directory is intended as a place for 3rd party XLA devices which are _not_
+integrated into the public repository.
+
+By adding entries to the BUILD target in this directory, a third party device
+can be included as a dependency of the JIT subsystem.
+
+For integration into the unit test system, see the files:
+
+- tensorflow/compiler/tests/plugin.bzl
+- tensorflow/compiler/xla/tests/plugin.bzl
+
+
+- 
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index ccab7bf348..7b7588f4ba 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -310,7 +310,7 @@ TEST_F(HloComputationTest, DeepCopyArrayAtIndices) {
 }
 
 TEST_F(HloComputationTest, DeepCopyTupleAtIndices) {
-  // Test that DeepCopyInstruction properly copies elements of a a tuple as
+  // Test that DeepCopyInstruction properly copies elements of a tuple as
   // specified by the given indices.
   auto builder = HloComputation::Builder(TestName());
   auto constant1 = builder.AddInstruction(HloInstruction::CreateConstant(
diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc
index 0682434bfb..6ea0f127d5 100644
--- a/tensorflow/compiler/xla/service/inliner.cc
+++ b/tensorflow/compiler/xla/service/inliner.cc
@@ -90,8 +90,12 @@ Status InlinerVisitor::HandleMap(
     // different than the map shape. Hence, a broadcast is needed, else the
     // cloned operand with new shape and operands work.
     if (root.opcode() != HloOpcode::kConstant) {
+      std::vector params;
+      for (int64 o = 0; o < root.operands().size(); o++) {
+        params.push_back(operands[root.operand(o)->parameter_number()]);
+      }
       HloInstruction* placed_instruction = computation_->AddInstruction(
-          root.CloneWithNewOperands(map->shape(), operands));
+          root.CloneWithNewOperands(map->shape(), params));
       TF_RETURN_IF_ERROR(
           computation_->ReplaceInstruction(map, placed_instruction));
     } else {
diff --git a/tensorflow/compiler/xla/service/inliner_test.cc b/tensorflow/compiler/xla/service/inliner_test.cc
index 9d845c5545..7aa1c7c835 100644
--- a/tensorflow/compiler/xla/service/inliner_test.cc
+++ b/tensorflow/compiler/xla/service/inliner_test.cc
@@ -108,5 +108,44 @@ TEST_F(InlinerTest, MapConstant) {
   LiteralTestUtil::ExpectEqual(*result, *expected);
 }
 
+TEST_F(InlinerTest, MapSubtractOppositeOrder) {
+  Shape r0f32 = ShapeUtil::MakeShape(F32, {});
+
+  // Note that the parameter ordinals are in the opposite order to their
+  // position as operands
+  auto max_builder = HloComputation::Builder(TestName());
+  auto param1 = max_builder.AddInstruction(
+          HloInstruction::CreateParameter(1, r0f32, "x"));
+  auto param2 = max_builder.AddInstruction(
+          HloInstruction::CreateParameter(0, r0f32, "y"));
+  max_builder.AddInstruction(HloInstruction::CreateBinary(
+          param1->shape(), HloOpcode::kSubtract, param1, param2));
+  auto max_f32 = max_builder.Build();
+
+  auto builder = HloComputation::Builder("MapSubFunction");
+  auto lhs = builder.AddInstruction(
+    HloInstruction::CreateConstant(Literal::CreateR1({1, 2, 3, 4})));
+  auto rhs = builder.AddInstruction(
+    HloInstruction::CreateConstant(Literal::CreateR1({4, 3, 2, 1})));
+  builder.AddInstruction(
+    HloInstruction::CreateMap(lhs->shape(), {lhs, rhs}, max_f32.get()));
+
+  auto computation = builder.Build();
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEmbeddedComputation(std::move(max_f32));
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  Inliner inliner;
+  EXPECT_TRUE(inliner.Run(hlo_module.get()).ValueOrDie());
+  EXPECT_THAT(hlo_module->entry_computation()->root_instruction(),
+          op::Subtract(rhs, lhs));
+
+  // Verify execution on CPU.
+  auto result = ExecuteAndTransfer(std::move(hlo_module), {});
+  auto expected = Literal::CreateR1({3, 1, -1, -3});
+  LiteralTestUtil::ExpectEqual(*result, *expected);
+}
+
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 22d7633ce2..a5057da9fd 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -191,7 +191,7 @@ def _ragged_split(tensor, pieces):
 
 
 def _ring_permutations(num_workers, num_subchunks, gpu_perm):
-  """"Generate an array of device index arrays, one for for each subchunk.
+  """"Generate an array of device index arrays, one for each subchunk.
 
   In the basic ring reduction algorithm there are size(T)/num_devices
   data chunks and each device process one chunk per tick, i.e. sending
diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md
index 9ce700f1a1..7d30032e53 100644
--- a/tensorflow/contrib/boosted_trees/README.md
+++ b/tensorflow/contrib/boosted_trees/README.md
@@ -1,7 +1,7 @@
 # TF Boosted Trees (TFBT)
 
 TF Boosted trees is an implementation of a gradient boosting algorithm with
-trees used as week learners.
+trees used as weak learners.
 
 ## Examples
 Folder "examples" demonstrates how TFBT estimators can be used for various
diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
index c003b1de66..47ee3d816f 100644
--- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
@@ -21,7 +21,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example.
   python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \
   --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \
   --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \
-  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1
 
   When training is done, accuracy on eval data is reported. Point tensorboard
   to the directory for the run to see how the training progresses:
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index 0539d77720..817c6eb3e1 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -22,7 +22,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example.
   python tensorflow/contrib/boosted_trees/examples/mnist.py \
   --output_dir="/tmp/mnist" --depth=4 --learning_rate=0.3 --batch_size=60000  \
   --examples_per_layer=60000 --eval_batch_size=10000 --num_eval_steps=1 \
-  --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+  --num_trees=10 --l2=1 --vmodule=training_ops=1
 
   When training is done, accuracy on eval data is reported. Point tensorboard
   to the directory for the run to see how the training progresses:
diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake
index e03026b1b0..8368898955 100644
--- a/tensorflow/contrib/cmake/external/cub.cmake
+++ b/tensorflow/contrib/cmake/external/cub.cmake
@@ -14,8 +14,8 @@
 # ==============================================================================
 include (ExternalProject)
 
-set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip)
-set(cub_HASH SHA256=b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe)
+set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip)
+set(cub_HASH SHA256=20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31)
 set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive)
diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
index d600d8c3c0..1e300e21df 100644
--- a/tensorflow/contrib/cmake/external/protobuf.cmake
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -15,8 +15,8 @@
 include (ExternalProject)
 
 set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
-set(PROTOBUF_URL https://github.com/mrry/protobuf.git)  # Includes MSVC fix.
-set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05)
+set(PROTOBUF_URL https://github.com/google/protobuf.git)
+set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9)
 
 if(WIN32)
   set(protobuf_STATIC_LIBRARIES 
diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 46c680aad5..65565aad7e 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -33,6 +33,8 @@ else(tensorflow_BUILD_ALL_KERNELS)
      "${tensorflow_source_dir}/tensorflow/core/kernels/matmul_op.cc"
      "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.h"
      "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.cc"
+     "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.h"
+     "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.cc"
      "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.h"
      "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.cc"
   )
@@ -65,6 +67,8 @@ if(tensorflow_BUILD_CONTRIB_KERNELS)
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc"
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 1d58b1d416..ac55b9ea92 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -179,6 +179,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
 
   # exclude the ones we don't want
   set(tf_test_src_py_exclude
+    # generally excluded
+    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
+
     # Python source line inspection tests are flaky on Windows (b/36375074).
     "${tensorflow_source_dir}/tensorflow/python/debug/cli/analyzer_cli_test.py"
     "${tensorflow_source_dir}/tensorflow/python/debug/cli/profile_analyzer_cli_test.py"
@@ -188,19 +191,16 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py"
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
     # generally not working
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/resource_variable_ops_test.py"
     "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py"
     # flaky test
     "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py"
+    # Fails because uses data dependencies with bazel
     "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py"
     # requires scipy
     "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py"
-    # flaky tests
+    # Takes very long to run without sharding (defined in bazel build file).
     "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"
-    "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py"
     # Loading resources in contrib doesn't seem to work on Windows
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py"
@@ -213,47 +213,57 @@ if (tensorflow_BUILD_PYTHON_TESTS)
   if (WIN32)
     set(tf_test_src_py_exclude
       ${tf_test_src_py_exclude}
-      # generally excluded
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
-
       # TODO: failing tests.
       # Nothing critical in here but should get this list down to []
       # The failing list is grouped by failure source
+
       # stl on windows handles overflows different
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/tensor_array_ops_test.py"  # Needs portpicker.
-      # Matrix_set_diag failing on GPU on windows.
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cholesky_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/diag_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_ops_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/ops/init_ops.py"
+      # Numerical issues, calculations off.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py"
+      # Float division by zero
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
+      # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/batch_matmul_op_test.py"
+      # Flaky because of local cluster creation.
+      "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
+      "${tensorflow_source_dir}tensorflow/python/training/localhost_cluster_performance_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"
+      # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
-      # misc
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py"
+      # IteratorGetMax OutOfRangeError
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reshape_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"  # Depends on gemmlowp -> pthread.
+      # Depends on gemmlowp -> pthread
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"
       # int32/int64 mixup
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py"
+      # Windows file management related issues.
+      "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
       # training tests
       "${tensorflow_source_dir}/tensorflow/python/training/basic_session_run_hooks_test.py"  # Needs tf.contrib fix.
       "${tensorflow_source_dir}/tensorflow/python/training/localhost_cluster_performance_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/quantize_training_test.py"  # Needs quantization ops to be included in windows.
       "${tensorflow_source_dir}/tensorflow/python/training/supervisor_test.py"  # Flaky I/O error on rename.
-      "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/server_lib_test.py"  # Test occasionally deadlocks.
-      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"  # Fails on multiple GPUs.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_grad_test.py"  # cudaSolver handle creation fails.
-
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops
       # Dataset tests
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on windows
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on Windows.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
       # Broken tensorboard test due to cmake issues.
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"  # Needs portpicker
@@ -264,8 +274,6 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py"  # Bad placement.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/topn_test.py"  # Results inaccurate
       "${tensorflow_source_dir}/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py"  # No libcurl support
-      # Newly running on Windows since TensorBoard backend move. Fail on Windows and need debug.
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on Windows.
       # Dask.Dataframe bugs on Window Build
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/tests/dataframe/tensorflow_dataframe_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py"
@@ -274,37 +282,15 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       # Need extra build
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/depthtospace_op_test.py"  # QuantizeV2
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/spacetodepth_op_test.py"  # QuantizeV2
       # Windows Path
       "${tensorflow_source_dir}/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py" #TODO: Fix path
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/models_test.py"
-      # Related to Windows Multiprocessing https://github.com/fchollet/keras/issues/5071
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/engine/training_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/utils/data_utils_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/callbacks_test.py"
-      # Scipy needed
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/image_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/binomial_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/chi2_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/geometric_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/inverse_gamma_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/logistic_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mixture_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_full_covariance_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_tril_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/poisson_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/quantized_distribution_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/relaxed_bernoulli_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/kmeans_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py"
-      # Failing with TF 1.3 (TODO)
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
+      # Numpy upgrade needed?
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py"
       # Test should only be run manually
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reduction_ops_test_big.py"
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 9e627bcaf4..1ce8954bb0 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -385,7 +385,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
       reset_op = state_ops.assign(
           opaque_params,
           array_ops.zeros(array_ops.shape(opaque_params), dtype=dtype))
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         val = saver.save(sess, save_path)
@@ -436,7 +436,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
       save_path = os.path.join(self.get_temp_dir(),
                                "save-restore-variable-test2")
       saver = saver_lib.Saver()
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         val = saver.save(sess, save_path)
@@ -484,7 +484,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
           array_ops.zeros(
               array_ops.shape(rnn.trainable_variables[0]), dtype=dtype))
 
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         inputs, initial_state = model.SynthesizeInput(seq_length, batch_size)
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 96447abd7c..5339ebb689 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -11,6 +11,9 @@ py_test(
     size = "small",
     srcs = ["batch_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958604
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
@@ -358,6 +361,9 @@ py_test(
     size = "small",
     srcs = ["sloppy_transformation_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958761
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index 6b0599ddd2..dd882acb8e 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -10,9 +10,8 @@ package(default_visibility = [
     "//tensorflow:__subpackages__",
 ])
 
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
@@ -27,6 +26,7 @@ tf_custom_op_py_library(
         "python/framework/experimental.py",
         "python/framework/tensor_util.py",
         "python/ops/__init__.py",
+        "python/ops/accumulate_n_v2.py",
         "python/ops/arg_scope.py",
         "python/ops/audio_ops.py",
         "python/ops/checkpoint_ops.py",
@@ -149,6 +149,31 @@ py_test(
     ],
 )
 
+py_test(
+    name = "accumulate_n_v2_test",
+    size = "small",
+    srcs = ["python/ops/accumulate_n_v2_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+    ],
+)
+
+py_test(
+    name = "accumulate_n_v2_eager_test",
+    size = "small",
+    srcs = ["python/ops/accumulate_n_v2_eager_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python/eager:backprop",
+    ],
+)
+
 py_test(
     name = "ops_test",
     size = "small",
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
new file mode 100644
index 0000000000..a0667bd489
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
@@ -0,0 +1,111 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ops that will eventually be folded into tensorflow/python/ops/math_ops.py
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import math_ops
+
+
+
+def accumulate_n_v2(inputs, shape=None, tensor_dtype=None, name=None):
+  """Returns the element-wise sum of a list of tensors.
+
+  Optionally, pass `shape` and `tensor_dtype` for shape and type checking,
+  otherwise, these are inferred.
+
+  `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+  wait for all of its inputs to be ready before beginning to sum. This can
+  save memory if inputs are ready at different times, since minimum temporary
+  storage is proportional to the output size rather than the inputs size.
+
+  Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+
+  For example:
+
+  ```python
+  a = tf.constant([[1, 2], [3, 4]])
+  b = tf.constant([[5, 0], [0, 6]])
+  tf.accumulate_n_v2([a, b, a])  # [[7, 4], [6, 14]]
+
+  # Explicitly pass shape and type
+  tf.accumulate_n_v2([a, b, a], shape=[2, 2], tensor_dtype=tf.int32)  
+                                                                   # [[7,  4],
+                                                                   #  [6, 14]]
+  ```
+
+  Args:
+    inputs: A list of `Tensor` objects, each with same shape and type.
+    shape: Shape of elements of `inputs`.
+    tensor_dtype: The type of `inputs`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of same shape and type as the elements of `inputs`.
+
+  Raises:
+    ValueError: If `inputs` don't all have same shape and dtype or the shape
+    cannot be inferred.
+  """
+  _INPUTS_ERR_MSG = ValueError("inputs must be a list of at least one Tensor"
+                               "with the same dtype and shape")
+  if not inputs or not isinstance(inputs, (list, tuple)):
+    raise _INPUTS_ERR_MSG
+  inputs = ops.convert_n_to_tensor_or_indexed_slices(inputs)
+  if not all(isinstance(x, ops.Tensor) for x in inputs):
+    raise _INPUTS_ERR_MSG
+  if not all(x.dtype == inputs[0].dtype for x in inputs):
+    raise _INPUTS_ERR_MSG
+  if shape is not None:
+    shape = tensor_shape.as_shape(shape)
+  else:
+    shape = tensor_shape.unknown_shape()
+  for input_tensor in inputs:
+    if isinstance(input_tensor, ops.Tensor):
+      shape = shape.merge_with(input_tensor.get_shape())
+
+  # tensor_dtype is for safety only; operator's output type computed in C++
+  if tensor_dtype is not None and tensor_dtype != inputs[0].dtype:
+    raise TypeError("tensor_dtype is {}, but input is of type {}"
+                    .format(tensor_dtype, inputs[0].dtype))
+
+  if len(inputs) == 1 and name is None:
+    return inputs[0]
+  elif len(inputs) == 1 and name is not None:
+    return array_ops.identity(inputs[0], name=name)
+  elif context.in_eager_mode():
+    # TemporaryVariable not currently supported in eager mode; fall back 
+    # onto AddN for now.
+    # TODO(frreiss) remove this once the lifetime of eager variables gets
+    # addressed
+    return math_ops.add_n(inputs, name=name)
+  else:
+    return gen_math_ops._accumulate_nv2(inputs, name=name, shape=shape)
+
+# The following code should eventually be merged into 
+# tensorflow/python/ops/math_grad.py
+@ops.RegisterGradient("AccumulateNV2")
+def _AddNGrad(op, grad):
+  """Same as gradient for AddN. Copies the gradient to all inputs."""
+  # Not broadcasting.
+  return [grad] * len(op.inputs)
+
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
new file mode 100644
index 0000000000..c2229bb8ad
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
@@ -0,0 +1,85 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for new version of accumulate_n op that will eventually go into 
+`ops.math_ops`.
+
+These test cases spefically exercise the `eager` APIs. They need to be in a 
+separate file from the remaining tests because eager mode is currently something
+you can turn on but can't turn off for the lifetime of the current process."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.framework.python.ops import accumulate_n_v2 as av2
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context as eager_context
+from tensorflow.python.eager import tape
+
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import test
+
+
+
+class AccumulateNV2EagerTest(test_util.TensorFlowTestCase):
+  """Tests of the new, differentiable version of accumulate_n"""
+
+  def testMinimalEagerMode(self):
+    forty = constant_op.constant(40)
+    two = constant_op.constant(2)
+    answer = av2.accumulate_n_v2([forty, two])
+    self.assertEqual(42, answer.numpy())
+
+
+  def testFloat(self):
+    np.random.seed(12345)
+    x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(sum(x), av2.accumulate_n_v2(tf_x).numpy())
+      self.assertAllClose(x[0] * 5, av2.accumulate_n_v2([tf_x[0]] * 5).numpy())
+
+  def testGrad(self):
+    np.random.seed(42)
+    num_inputs = 3
+    input_vars = [
+        resource_variable_ops.ResourceVariable(10.0 * np.random.random(), 
+                                               name="t%d" % i)
+        for i in range(0, num_inputs)
+    ]
+
+    def fn(first, second, third):
+      return av2.accumulate_n_v2([first, second, third])
+
+    grad_fn = backprop.gradients_function(fn)      
+    grad = grad_fn(input_vars[0], input_vars[1], input_vars[2])
+    self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
+                        [elem.numpy() for elem in grad])
+
+
+
+if __name__ == "__main__":
+  ops.enable_eager_execution()
+  test.main()
+
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
new file mode 100644
index 0000000000..3386e849d5
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
@@ -0,0 +1,123 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for new version of accumulate_n op that will eventually go into 
+`ops.math_ops`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.framework.python.ops import accumulate_n_v2 as av2
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+
+
+class AccumulateNV2Test(test_util.TensorFlowTestCase):
+  """Tests of the new, differentiable version of accumulate_n"""
+
+  def testFloat(self):
+    np.random.seed(12345)
+    x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(sum(x), av2.accumulate_n_v2(tf_x).eval())
+      self.assertAllClose(x[0] * 5, av2.accumulate_n_v2([tf_x[0]] * 5).eval())
+
+  def testInt(self):
+    np.random.seed(54321)
+    x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllEqual(sum(x), av2.accumulate_n_v2(tf_x).eval())
+      self.assertAllEqual(x[0] * 6, av2.accumulate_n_v2([tf_x[0]] * 6).eval())
+
+  def testGrad(self):
+    np.random.seed(42)
+    for num_inputs in range(1, 10):
+      with self.test_session(use_gpu=True) as sess:
+        input_vars = [
+            variables.Variable(10.0 * np.random.random())
+            for i in range(0, num_inputs)
+        ]
+        accum_n = av2.accumulate_n_v2(input_vars)
+        sess.run(variables.global_variables_initializer())
+        accum_n_grad = gradients.gradients(accum_n, input_vars)
+        self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
+                            [g.eval() for g in accum_n_grad])
+
+  # The tests below used to be in a separate class under cwise_ops_test.py,
+  # which did not run in the default test target.
+  # Putting them here so that everything that exercises AccumulateNV2 is in
+  # one place and the default build runs all unit tests.
+  def testSimple(self):
+    with self.test_session():
+      random_arrays = [
+          np.random.rand(16, 16, 16, 16).astype(np.float32) for _ in range(20)
+      ]
+      random_tensors = [
+          ops.convert_to_tensor(
+              x, dtype=dtypes_lib.float32) for x in random_arrays
+      ]
+      tf_val = av2.accumulate_n_v2(random_tensors)
+      np_val = random_arrays[0]
+      for random_array in random_arrays[1:]:
+        np_val += random_array
+      self.assertAllClose(np_val, tf_val.eval())
+
+  def testZeroArgs(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        tf_val = av2.accumulate_n_v2([])
+        tf_val.eval()
+
+  def testWrongShape(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        a = variables.Variable(0.2)
+        b = variables.Variable(0.1)
+        tf_val = av2.accumulate_n_v2([a,b], shape=[2,2]) # Should be shape=[]
+
+  def testIncompatibleShapes(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        a = variables.Variable(np.array([0.1,0.2]))
+        b = variables.Variable(np.array([[0.3],[0.4]]))
+        tf_val = av2.accumulate_n_v2([a,b]) 
+
+  def testWrongType(self):
+    with self.test_session():
+      with self.assertRaises(TypeError):
+        a = variables.Variable(0.2, dtype=np.float32)
+        b = variables.Variable(0.1, dtype=np.float32)
+        tf_val = av2.accumulate_n_v2([a,b], tensor_dtype=np.int32) 
+
+  def testWrongTypeOneInput(self):
+    # Scenario that used to trigger a bug, even when testWrongType() worked
+    with self.test_session():
+      with self.assertRaises(TypeError):
+        a = variables.Variable(0.2, dtype=np.float32)
+        tf_val = av2.accumulate_n_v2([a], tensor_dtype=np.int32) 
+
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py
index 59a322d3ca..d030dffade 100755
--- a/tensorflow/contrib/image/__init__.py
+++ b/tensorflow/contrib/image/__init__.py
@@ -26,6 +26,8 @@ projective transforms (including rotation) are supported.
 @@random_yiq_hsv
 @@rotate
 @@transform
+@@translate
+@@translations_to_projective_transforms
 @@bipartite_match
 @@single_image_random_dot_stereograms
 """
@@ -41,6 +43,8 @@ from tensorflow.contrib.image.python.ops.image_ops import angles_to_projective_t
 from tensorflow.contrib.image.python.ops.image_ops import compose_transforms
 from tensorflow.contrib.image.python.ops.image_ops import rotate
 from tensorflow.contrib.image.python.ops.image_ops import transform
+from tensorflow.contrib.image.python.ops.image_ops import translate
+from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms
 from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
index b8a0706b61..b50177ae56 100644
--- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
+++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
@@ -36,8 +36,8 @@ _DTYPES = set(
 class ImageOpsTest(test_util.TensorFlowTestCase):
 
   def test_zeros(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         for shape in [(5, 5), (24, 24), (2, 24, 24, 3)]:
           for angle in [0, 1, np.pi / 2.0]:
             image = array_ops.zeros(shape, dtype)
@@ -46,8 +46,8 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                 np.zeros(shape, dtype.as_numpy_dtype()))
 
   def test_rotate_even(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = array_ops.reshape(
             math_ops.cast(math_ops.range(36), dtype), (6, 6))
         image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1])
@@ -68,8 +68,8 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                               [1, 7, 13, 19, 25, 31], [0, 6, 12, 18, 24, 30]]])
 
   def test_rotate_odd(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = array_ops.reshape(
             math_ops.cast(math_ops.range(25), dtype), (5, 5))
         image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1])
@@ -87,9 +87,25 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                               [22, 17, 12, 7, 2], [23, 18, 13, 8, 3],
                               [24, 19, 14, 9, 4]]])
 
+  def test_translate(self):
+    for dtype in _DTYPES:
+      with self.test_session():
+        image = constant_op.constant(
+            [[1, 0, 1, 0],
+             [0, 1, 0, 1],
+             [1, 0, 1, 0],
+             [0, 1, 0, 1]], dtype=dtype)
+        translation = constant_op.constant([-1, -1], dtypes.float32)
+        image_translated = image_ops.translate(image, translation)
+        self.assertAllEqual(image_translated.eval(),
+                            [[1, 0, 1, 0],
+                             [0, 1, 0, 0],
+                             [1, 0, 1, 0],
+                             [0, 0, 0, 0]])
+
   def test_compose(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = constant_op.constant(
             [[1, 1, 1, 0],
              [1, 0, 0, 0],
@@ -246,4 +262,3 @@ class BipartiteMatchTest(test_util.TensorFlowTestCase):
 
 if __name__ == "__main__":
   googletest.main()
-
diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py
index aef3e385b5..011ddeaa9a 100644
--- a/tensorflow/contrib/image/python/ops/image_ops.py
+++ b/tensorflow/contrib/image/python/ops/image_ops.py
@@ -37,16 +37,18 @@ _IMAGE_DTYPES = set(
 ops.RegisterShape("ImageProjectiveTransform")(common_shapes.call_cpp_shape_fn)
 
 
-def rotate(images, angles, interpolation="NEAREST"):
+def rotate(images, angles, interpolation="NEAREST", name=None):
   """Rotate image(s) by the passed angle(s) in radians.
 
   Args:
     images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
-       (num_rows, num_columns) (HW).
+       (num_rows, num_columns) (HW). The rank must be statically known (the
+       shape is not `TensorShape(None)`.
     angles: A scalar angle to rotate all images by, or (if images has rank 4)
        a vector of length num_images, with an angle for each image in the batch.
     interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR".
+    name: The name of the op.
 
   Returns:
     Image(s) with the same type and shape as `images`, rotated by the given
@@ -55,38 +57,77 @@ def rotate(images, angles, interpolation="NEAREST"):
   Raises:
     TypeError: If `image` is an invalid type.
   """
-  image_or_images = ops.convert_to_tensor(images, name="images")
-  if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
-    raise TypeError("Invalid dtype %s." % image_or_images.dtype)
-  if len(image_or_images.get_shape()) == 2:
-    images = image_or_images[None, :, :, None]
-  elif len(image_or_images.get_shape()) == 3:
-    images = image_or_images[None, :, :, :]
-  elif len(image_or_images.get_shape()) == 4:
-    images = image_or_images
-  else:
-    raise TypeError("Images should have rank between 2 and 4.")
-
-  image_height = math_ops.cast(array_ops.shape(images)[1], dtypes.float32)[None]
-  image_width = math_ops.cast(array_ops.shape(images)[2], dtypes.float32)[None]
-  output = transform(
-      images,
-      angles_to_projective_transforms(angles, image_height, image_width),
-      interpolation=interpolation)
-  if len(image_or_images.get_shape()) == 2:
-    return output[0, :, :, 0]
-  elif len(image_or_images.get_shape()) == 3:
-    return output[0, :, :, :]
-  else:
-    return output
+  with ops.name_scope(name, "rotate"):
+    image_or_images = ops.convert_to_tensor(images)
+    if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
+      raise TypeError("Invalid dtype %s." % image_or_images.dtype)
+    elif image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      images = image_or_images[None, :, :, None]
+    elif len(image_or_images.get_shape()) == 3:
+      images = image_or_images[None, :, :, :]
+    elif len(image_or_images.get_shape()) == 4:
+      images = image_or_images
+    else:
+      raise TypeError("Images should have rank between 2 and 4.")
+
+    image_height = math_ops.cast(array_ops.shape(images)[1],
+                                 dtypes.float32)[None]
+    image_width = math_ops.cast(array_ops.shape(images)[2],
+                                dtypes.float32)[None]
+    output = transform(
+        images,
+        angles_to_projective_transforms(angles, image_height, image_width),
+        interpolation=interpolation)
+    if image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      return output[0, :, :, 0]
+    elif len(image_or_images.get_shape()) == 3:
+      return output[0, :, :, :]
+    else:
+      return output
+
+
+def translate(images, translations, interpolation="NEAREST", name=None):
+  """Translate image(s) by the passed vectors(s).
 
+  Args:
+    images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
+        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
+        (num_rows, num_columns) (HW). The rank must be statically known (the
+        shape is not `TensorShape(None)`.
+    translations: A vector representing [dx, dy] or (if images has rank 4)
+        a matrix of length num_images, with a [dx, dy] vector for each image in
+        the batch.
+    interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR".
+    name: The name of the op.
 
-def angles_to_projective_transforms(angles, image_height, image_width):
+  Returns:
+    Image(s) with the same type and shape as `images`, translated by the given
+        vector(s). Empty space due to the translation will be filled with zeros.
+
+  Raises:
+    TypeError: If `image` is an invalid type.
+  """
+  with ops.name_scope(name, "translate"):
+    return transform(
+        images,
+        translations_to_projective_transforms(translations),
+        interpolation=interpolation)
+
+
+def angles_to_projective_transforms(angles,
+                                    image_height,
+                                    image_width,
+                                    name=None):
   """Returns projective transform(s) for the given angle(s).
 
   Args:
     angles: A scalar angle to rotate all images by, or (for batches of images)
-      a vector with an angle to rotate each image in the batch.
+        a vector with an angle to rotate each image in the batch. The rank must
+        be statically known (the shape is not `TensorShape(None)`.
     image_height: Height of the image(s) to be transformed.
     image_width: Width of the image(s) to be transformed.
 
@@ -94,41 +135,89 @@ def angles_to_projective_transforms(angles, image_height, image_width):
     A tensor of shape (num_images, 8). Projective transforms which can be given
       to `tf.contrib.image.transform`.
   """
-  angle_or_angles = ops.convert_to_tensor(
-      angles, name="angles", dtype=dtypes.float32)
-  if len(angle_or_angles.get_shape()) == 0:  # pylint: disable=g-explicit-length-test
-    angles = angle_or_angles[None]
-  elif len(angle_or_angles.get_shape()) == 1:
-    angles = angle_or_angles
-  else:
-    raise TypeError("Angles should have rank 0 or 1.")
-  x_offset = ((image_width - 1) - (math_ops.cos(angles) *
-                                   (image_width - 1) - math_ops.sin(angles) *
-                                   (image_height - 1))) / 2.0
-  y_offset = ((image_height - 1) - (math_ops.sin(angles) *
-                                    (image_width - 1) + math_ops.cos(angles) *
-                                    (image_height - 1))) / 2.0
-  num_angles = array_ops.shape(angles)[0]
-  return array_ops.concat(
-      values=[
-          math_ops.cos(angles)[:, None],
-          -math_ops.sin(angles)[:, None],
-          x_offset[:, None],
-          math_ops.sin(angles)[:, None],
-          math_ops.cos(angles)[:, None],
-          y_offset[:, None],
-          array_ops.zeros((num_angles, 2), dtypes.float32),
-      ],
-      axis=1)
-
-
-def transform(images, transforms, interpolation="NEAREST"):
+  with ops.name_scope(name, "angles_to_projective_transforms"):
+    angle_or_angles = ops.convert_to_tensor(
+        angles, name="angles", dtype=dtypes.float32)
+    if len(angle_or_angles.get_shape()) == 0:  # pylint: disable=g-explicit-length-test
+      angles = angle_or_angles[None]
+    elif len(angle_or_angles.get_shape()) == 1:
+      angles = angle_or_angles
+    else:
+      raise TypeError("Angles should have rank 0 or 1.")
+    x_offset = ((image_width - 1) - (math_ops.cos(angles) *
+                                     (image_width - 1) - math_ops.sin(angles) *
+                                     (image_height - 1))) / 2.0
+    y_offset = ((image_height - 1) - (math_ops.sin(angles) *
+                                      (image_width - 1) + math_ops.cos(angles) *
+                                      (image_height - 1))) / 2.0
+    num_angles = array_ops.shape(angles)[0]
+    return array_ops.concat(
+        values=[
+            math_ops.cos(angles)[:, None],
+            -math_ops.sin(angles)[:, None],
+            x_offset[:, None],
+            math_ops.sin(angles)[:, None],
+            math_ops.cos(angles)[:, None],
+            y_offset[:, None],
+            array_ops.zeros((num_angles, 2), dtypes.float32),
+        ],
+        axis=1)
+
+
+def translations_to_projective_transforms(translations, name=None):
+  """Returns projective transform(s) for the given translation(s).
+
+  Args:
+      translations: A 2-element list representing [dx, dy] or a matrix of
+          2-element lists representing [dx, dy] to translate for each image
+          (for a batch of images). The rank must be statically known (the shape
+          is not `TensorShape(None)`.
+      name: The name of the op.
+
+  Returns:
+      A tensor of shape (num_images, 8) projective transforms which can be given
+          to `tf.contrib.image.transform`.
+  """
+  with ops.name_scope(name, "translations_to_projective_transforms"):
+    translation_or_translations = ops.convert_to_tensor(
+        translations, name="translations", dtype=dtypes.float32)
+    if translation_or_translations.get_shape().ndims is None:
+      raise TypeError(
+          "translation_or_translations rank must be statically known")
+    elif len(translation_or_translations.get_shape()) == 1:
+      translations = translation_or_translations[None]
+    elif len(translation_or_translations.get_shape()) == 2:
+      translations = translation_or_translations
+    else:
+      raise TypeError("Translations should have rank 1 or 2.")
+    num_translations = array_ops.shape(translations)[0]
+    # The translation matrix looks like:
+    #     [[1 0 -dx]
+    #      [0 1 -dy]
+    #      [0 0 1]]
+    # where the last entry is implicit.
+    # Translation matrices are always float32.
+    return array_ops.concat(
+        values=[
+            array_ops.ones((num_translations, 1), dtypes.float32),
+            array_ops.zeros((num_translations, 1), dtypes.float32),
+            -translations[:, 0, None],
+            array_ops.zeros((num_translations, 1), dtypes.float32),
+            array_ops.ones((num_translations, 1), dtypes.float32),
+            -translations[:, 1, None],
+            array_ops.zeros((num_translations, 2), dtypes.float32),
+        ],
+        axis=1)
+
+
+def transform(images, transforms, interpolation="NEAREST", name=None):
   """Applies the given transform(s) to the image(s).
 
   Args:
     images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
-       (num_rows, num_columns) (HW).
+       (num_rows, num_columns) (HW). The rank must be statically known (the
+       shape is not `TensorShape(None)`.
     transforms: Projective transform matrix/matrices. A vector of length 8 or
        tensor of size N x 8. If one row of transforms is
        [a0, a1, a2, b0, b1, b2, c0, c1], then it maps the *output* point
@@ -146,34 +235,40 @@ def transform(images, transforms, interpolation="NEAREST"):
   Raises:
     TypeError: If `image` is an invalid type.
   """
-  image_or_images = ops.convert_to_tensor(images, name="images")
-  transform_or_transforms = ops.convert_to_tensor(
-      transforms, name="transforms", dtype=dtypes.float32)
-  if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
-    raise TypeError("Invalid dtype %s." % image_or_images.dtype)
-  if len(image_or_images.get_shape()) == 2:
-    images = image_or_images[None, :, :, None]
-  elif len(image_or_images.get_shape()) == 3:
-    images = image_or_images[None, :, :, :]
-  elif len(image_or_images.get_shape()) == 4:
-    images = image_or_images
-  else:
-    raise TypeError("Images should have rank between 2 and 4.")
-
-  if len(transform_or_transforms.get_shape()) == 1:
-    transforms = transform_or_transforms[None]
-  elif len(transform_or_transforms.get_shape()) == 2:
-    transforms = transform_or_transforms
-  else:
-    raise TypeError("Transforms should have rank 1 or 2.")
-  output = gen_image_ops.image_projective_transform(
-      images, transforms, interpolation=interpolation.upper())
-  if len(image_or_images.get_shape()) == 2:
-    return output[0, :, :, 0]
-  elif len(image_or_images.get_shape()) == 3:
-    return output[0, :, :, :]
-  else:
-    return output
+  with ops.name_scope(name, "transform"):
+    image_or_images = ops.convert_to_tensor(images, name="images")
+    transform_or_transforms = ops.convert_to_tensor(
+        transforms, name="transforms", dtype=dtypes.float32)
+    if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
+      raise TypeError("Invalid dtype %s." % image_or_images.dtype)
+    elif image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      images = image_or_images[None, :, :, None]
+    elif len(image_or_images.get_shape()) == 3:
+      images = image_or_images[None, :, :, :]
+    elif len(image_or_images.get_shape()) == 4:
+      images = image_or_images
+    else:
+      raise TypeError("Images should have rank between 2 and 4.")
+
+    if len(transform_or_transforms.get_shape()) == 1:
+      transforms = transform_or_transforms[None]
+    elif transform_or_transforms.get_shape().ndims is None:
+      raise TypeError(
+          "transform_or_transforms rank must be statically known")
+    elif len(transform_or_transforms.get_shape()) == 2:
+      transforms = transform_or_transforms
+    else:
+      raise TypeError("Transforms should have rank 1 or 2.")
+    output = gen_image_ops.image_projective_transform(
+        images, transforms, interpolation=interpolation.upper())
+    if len(image_or_images.get_shape()) == 2:
+      return output[0, :, :, 0]
+    elif len(image_or_images.get_shape()) == 3:
+      return output[0, :, :, :]
+    else:
+      return output
 
 
 def compose_transforms(*transforms):
@@ -191,11 +286,12 @@ def compose_transforms(*transforms):
         order.
   """
   assert transforms, "transforms cannot be empty"
-  composed = _flat_transforms_to_matrices(transforms[0])
-  for tr in transforms[1:]:
-    # Multiply batches of matrices.
-    composed = math_ops.matmul(composed, _flat_transforms_to_matrices(tr))
-  return _transform_matrices_to_flat(composed)
+  with ops.name_scope("compose_transforms"):
+    composed = _flat_transforms_to_matrices(transforms[0])
+    for tr in transforms[1:]:
+      # Multiply batches of matrices.
+      composed = math_ops.matmul(composed, _flat_transforms_to_matrices(tr))
+    return _transform_matrices_to_flat(composed)
 
 
 def _flat_transforms_to_matrices(transforms):
@@ -211,8 +307,8 @@ def _flat_transforms_to_matrices(transforms):
 
 def _transform_matrices_to_flat(transform_matrices):
   # Flatten each matrix.
-  transforms = array_ops.reshape(
-      transform_matrices, constant_op.constant([-1, 9]))
+  transforms = array_ops.reshape(transform_matrices,
+                                 constant_op.constant([-1, 9]))
   # Divide each matrix by the last entry (normally 1).
   transforms /= transforms[:, 8:9]
   return transforms[:, :8]
@@ -260,10 +356,10 @@ def _image_projective_transform_grad(op, grad):
     return [output, None]
 
 
-def bipartite_match(
-    distance_mat,
-    num_valid_rows,
-    top_k=-1):
+def bipartite_match(distance_mat,
+                    num_valid_rows,
+                    top_k=-1,
+                    name="bipartite_match"):
   """Find bipartite matching based on a given distance matrix.
 
   A greedy bi-partite matching algorithm is used to obtain the matching with
@@ -282,6 +378,7 @@ def bipartite_match(
     top_k: A scalar that specifies the number of top-k matches to retrieve.
       If set to be negative, then is set according to the maximum number of
       matches from `distance_mat`.
+    name: The name of the op.
 
   Returns:
     row_to_col_match_indices: A vector of length num_rows, which is the number
@@ -292,7 +389,8 @@ def bipartite_match(
       If `col_to_row_match_indices[j]` is not -1, column j is matched to row
       `col_to_row_match_indices[j]`.
   """
-  result = gen_image_ops.bipartite_match(distance_mat, num_valid_rows, top_k)
+  result = gen_image_ops.bipartite_match(
+      distance_mat, num_valid_rows, top_k, name=name)
   return result
 
 
diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py
index 0b5c3d4928..69d97f0b5b 100644
--- a/tensorflow/contrib/kfac/python/ops/loss_functions.py
+++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py
@@ -104,7 +104,7 @@ class LossFunction(object):
 
   @abc.abstractmethod
   def multiply_hessian_factor_transpose(self, vector):
-    """Right-multiply a vector by the tranpose of a factor B of the Hessian.
+    """Right-multiply a vector by the transpose of a factor B of the Hessian.
 
     Here the 'Hessian' is the Hessian matrix (i.e. matrix of 2nd-derivatives)
     of the loss function with respect to its inputs.  Typically this will be
@@ -218,7 +218,7 @@ class NegativeLogProbLoss(LossFunction):
 
   @abc.abstractmethod
   def multiply_fisher_factor_transpose(self, vector):
-    """Right-multiply a vector by the tranpose of a factor B of the Fisher.
+    """Right-multiply a vector by the transpose of a factor B of the Fisher.
 
     Here the 'Fisher' is the Fisher information matrix (i.e. expected outer-
     product of gradients) with respect to the parameters of the underlying
@@ -397,7 +397,7 @@ class NormalMeanVarianceNegativeLogProbLoss(DistributionNegativeLogProbLoss):
 
   This class parameterizes a multivariate normal distribution with n independent
   dimensions. Unlike `NormalMeanNegativeLogProbLoss`, this class does not
-  assume the variance is held constant. The Fisher Information for for n = 1
+  assume the variance is held constant. The Fisher Information for n = 1
   is given by,
 
   F = [[1 / variance,                0],
diff --git a/tensorflow/contrib/kfac/python/ops/op_queue.py b/tensorflow/contrib/kfac/python/ops/op_queue.py
index 0617c5be4d..831870fca4 100644
--- a/tensorflow/contrib/kfac/python/ops/op_queue.py
+++ b/tensorflow/contrib/kfac/python/ops/op_queue.py
@@ -61,7 +61,7 @@ class OpQueue(object):
       sess: tf.Session.
 
     Returns:
-      Next Op chosen from from 'ops'.
+      Next Op chosen from 'ops'.
     """
     # In Python 3, type(next_op_name) == bytes. Calling bytes.decode('ascii')
     # returns a str.
diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py
index d8ab7c2d70..d309ba958d 100644
--- a/tensorflow/contrib/layers/__init__.py
+++ b/tensorflow/contrib/layers/__init__.py
@@ -47,6 +47,7 @@ See the @{$python/contrib.layers} guide.
 @@separable_conv2d
 @@separable_convolution2d
 @@softmax
+@@spatial_softmax
 @@stack
 @@unit_norm
 @@bow_encoder
diff --git a/tensorflow/contrib/learn/python/learn/learn_runner.py b/tensorflow/contrib/learn/python/learn/learn_runner.py
index 9f9740ec49..2af723a0d6 100644
--- a/tensorflow/contrib/learn/python/learn/learn_runner.py
+++ b/tensorflow/contrib/learn/python/learn/learn_runner.py
@@ -165,7 +165,7 @@ def run(experiment_fn, output_dir=None, schedule=None, run_config=None,
       must be None.
       2) It accepts two arguments `run_config` and `hparams`, which should be
       used to create the `Estimator` (`run_config` passed as `config` to its
-      constructor; `hparams` used as the hyper-paremeters of the model).
+      constructor; `hparams` used as the hyper-parameters of the model).
       It must return an `Experiment`. For this case, `output_dir` must be None.
     output_dir: Base output directory [Deprecated].
     schedule: The name of the method in the `Experiment` to run.
diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py
index 1d2477b8b7..7c523ad492 100644
--- a/tensorflow/contrib/losses/python/losses/loss_ops.py
+++ b/tensorflow/contrib/losses/python/losses/loss_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.util.deprecation import deprecated
+from tensorflow.python.util.deprecation import deprecated_args
 
 __all__ = ["absolute_difference",
            "add_loss",
@@ -623,8 +624,9 @@ def mean_pairwise_squared_error(
 
 
 @deprecated("2016-12-30", "Use tf.losses.cosine_distance instead.")
+@deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def cosine_distance(
-    predictions, labels=None, dim=None, weights=1.0, scope=None):
+    predictions, labels=None, axis=None, weights=1.0, scope=None, dim=None):
   """Adds a cosine-distance loss to the training procedure.
 
   Note that the function assumes that `predictions` and `labels` are already
@@ -633,10 +635,11 @@ def cosine_distance(
   Args:
     predictions: An arbitrary matrix.
     labels: A `Tensor` whose shape matches 'predictions'
-    dim: The dimension along which the cosine distance is computed.
+    axis: The dimension along which the cosine distance is computed.
     weights: Coefficients for the loss a scalar, a tensor of shape
       [batch_size] or a tensor whose shape matches `predictions`.
     scope: The scope for the operations performed in computing the loss.
+    dim: The old (deprecated) name for `axis`.
 
   Returns:
     A scalar `Tensor` representing the loss value.
@@ -645,8 +648,12 @@ def cosine_distance(
     ValueError: If `predictions` shape doesn't match `labels` shape, or
       `weights` is `None`.
   """
-  if dim is None:
-    raise ValueError("`dim` cannot be None.")
+  if dim is not None:
+    if axis is not None:
+      raise ValueError("Cannot specify both 'axis' and 'dim'")
+    axis = dim
+  if axis is None and dim is None:
+    raise ValueError("You must specify 'axis'.")
   with ops.name_scope(scope, "cosine_distance_loss",
                       [predictions, labels, weights]) as scope:
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
@@ -655,5 +662,5 @@ def cosine_distance(
     labels = math_ops.to_float(labels)
 
     radial_diffs = math_ops.multiply(predictions, labels)
-    losses = 1 - math_ops.reduce_sum(radial_diffs, reduction_indices=[dim,])
+    losses = 1 - math_ops.reduce_sum(radial_diffs, reduction_indices=[axis,])
     return compute_weighted_loss(losses, weights, scope=scope)
diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 81024c26a4..b582493131 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -194,6 +194,10 @@ LIBFLAGS :=
 
 # If we're on OS X, make sure that globals aren't stripped out.
 ifeq ($(TARGET),OSX)
+ifeq ($(HAS_GEN_HOST_PROTOC),true)
+	LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib
+	export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib
+endif
 	LDFLAGS += -all_load
 endif
 # Make sure that we don't strip global constructors on Linux.
diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index f0b9658e3d..12e3f58930 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -54,7 +54,7 @@ download_and_extract() {
   elif [[ "${url}" == *zip ]]; then
     tempdir=$(mktemp -d)
     tempdir2=$(mktemp -d)
-    wget ${url} -P ${tempdir}
+    wget -P ${tempdir} ${url}
     unzip ${tempdir}/* -d ${tempdir2}
     # unzip has no strip components, so unzip to a temp dir, and move the files
     # we want from the tempdir to destination.
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index a8690a04ad..8b77c99cb5 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -264,3 +264,4 @@ tensorflow/core/kernels/spacetobatch_functor.cc
 tensorflow/core/kernels/spacetobatch_op.cc
 tensorflow/core/kernels/batchtospace_op.cc
 tensorflow/core/kernels/warn_about_ints.cc
+tensorflow/core/kernels/segment_reduction_ops.cc
diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
index 303c02dfa4..2932ae1c8d 100644
--- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
+++ b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
@@ -749,7 +749,7 @@ def meta_graph_transform(
         base_meta_graph_def, meta_graph_def, collection_name,
         removed_op_names)
 
-  # Append newly added initalizers to collection.
+  # Append newly added initializers to collection.
   _add_new_inits_to_collection(meta_graph_def, updated_initializer_names)
 
   # Copy signature_defs, excluding any pruned nodes
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index f24bec7f11..6e038481e3 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -5856,7 +5856,7 @@ class StreamingMeanIOUTest(test.TestCase):
       sess.run(variables.local_variables_initializer())
       for _ in range(5):
         sess.run(update_op)
-      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0, 0.])
+      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
   def testUpdateOpEvalIsAccumulatedConfusionMatrix(self):
@@ -5938,6 +5938,58 @@ class StreamingMeanIOUTest(test.TestCase):
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  def testMissingClassInLabels(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 2, 1, 1, 0],
+       [0, 1, 2, 2, 0, 1]],
+      [[0, 0, 2, 1, 1, 1],
+       [1, 1, 2, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
+        miou.eval())
+
+  def testMissingClassOverallSmall(self):
+    labels = constant_op.constant([0])
+    predictions = constant_op.constant([0])
+    num_classes = 2
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
+      self.assertAlmostEqual(1, miou.eval())
+
+  def testMissingClassOverallLarge(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 1, 0, 0, 1, 1]],
+      [[0, 0, 0, 1, 1, 1],
+       [1, 1, 1, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
+
 
 class StreamingConcatTest(test.TestCase):
 
diff --git a/tensorflow/contrib/mpi_collectives/__init__.py b/tensorflow/contrib/mpi_collectives/__init__.py
index b94f7b0a35..9ed16a6f07 100644
--- a/tensorflow/contrib/mpi_collectives/__init__.py
+++ b/tensorflow/contrib/mpi_collectives/__init__.py
@@ -194,7 +194,7 @@ class DistributedOptimizer(tf.train.Optimizer):
 
     See Optimizer.compute_gradients() for more info.
 
-    In DistributedOptimizer, compute_gradients() is overriden to also
+    In DistributedOptimizer, compute_gradients() is overridden to also
     allreduce the gradients before returning them.
     """
     gradients = (super(DistributedOptimizer, self)
diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py
index 7007e26bac..3bf795d19a 100644
--- a/tensorflow/contrib/nn/__init__.py
+++ b/tensorflow/contrib/nn/__init__.py
@@ -18,6 +18,7 @@
 @@deprecated_flipped_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sigmoid_cross_entropy_with_logits
+@@nth_element
 @@rank_sampled_softmax_loss
 @@scaled_softplus
 """
@@ -31,6 +32,7 @@ from tensorflow.contrib.nn.python.ops.alpha_dropout import *
 from tensorflow.contrib.nn.python.ops.cross_entropy import *
 from tensorflow.contrib.nn.python.ops.sampling_ops import *
 from tensorflow.contrib.nn.python.ops.scaled_softplus import *
+from tensorflow.python.ops.nn_ops import nth_element
 # pylint: enable=unused-import,wildcard-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field.py b/tensorflow/contrib/receptive_field/python/util/receptive_field.py
index db190a1a41..8b34465d21 100644
--- a/tensorflow/contrib/receptive_field/python/util/receptive_field.py
+++ b/tensorflow/contrib/receptive_field/python/util/receptive_field.py
@@ -27,13 +27,15 @@ import math
 from tensorflow.contrib.receptive_field.python.util import graph_compute_order
 from tensorflow.contrib.util import make_ndarray
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.framework import ops as framework_ops
+import numpy as np
 
 # White-listed layer operations, which do not affect the receptive field
 # computation.
 _UNCHANGED_RF_LAYER_OPS = [
-    "Softplus", "Relu", "BiasAdd", "Mul", "Add", "Const", "Identity",
-    "VariableV2", "Sub", "Rsqrt", "ConcatV2"
-]
+  'Add', 'BiasAdd', 'Ceil', 'ConcatV2', 'Const', 'Floor', 'Identity', 'Log',
+  'Mul', 'Pow', 'RealDiv', 'Relu', 'Round', 'Rsqrt', 'Softplus', 'Sub',
+  'VariableV2']
 
 # Different ways in which padding modes may be spelled.
 _VALID_PADDING = ["VALID", b"VALID"]
@@ -238,7 +240,8 @@ def _get_layer_params(node, name_to_order_node):
     padding_x = 0
     padding_y = 0
   else:
-    raise ValueError("Unknown layer op: %s" % node.op)
+    raise ValueError("Unknown layer for operation '%s': %s" %
+                     (node.name, node.op))
   return kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, padding_y
 
 
@@ -304,13 +307,103 @@ def _get_effective_padding_node_input(stride, padding,
   return stride * effective_padding_output + padding
 
 
-def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
-  """Computes receptive field (RF) parameters from a GraphDef object.
+class ReceptiveField:
+  """
+  Receptive field of a convolutional neural network.
+
+  Args:
+    size: Receptive field size.
+    stride: Effective stride.
+    padding: Effective padding.
+  """
+  def __init__(self, size, stride, padding):
+    self.size = np.asarray(size)
+    self.stride = np.asarray(stride)
+    self.padding = np.asarray(padding)
+
+  def compute_input_center_coordinates(self, y, axis=None):
+    """
+    Computes the center of the receptive field that generated a feature.
+
+    Args:
+      y: An array of feature coordinates with shape `(..., d)`, where `d` is the
+        number of dimensions of the coordinates.
+      axis: The dimensions for which to compute the input center coordinates.
+        If `None` (the default), compute the input center coordinates for all
+        dimensions.
+
+    Returns:
+      x: Center of the receptive field that generated the features, at the input
+        of the network.
+
+    Raises:
+      ValueError: If the number of dimensions of the feature coordinates does
+        not match the number of elements in `axis`.
+    """
+    # Use all dimensions.
+    if axis is None:
+      axis = range(self.size.size)
+    # Ensure axis is a list because tuples have different indexing behavior.
+    axis = list(axis)
+    y = np.asarray(y)
+    if y.shape[-1] != len(axis):
+      raise ValueError("Dimensionality of the feature coordinates `y` (%d) "
+                       "does not match dimensionality of `axis` (%d)" %
+                       (y.shape[-1], len(axis)))
+    return - self.padding[axis] + y * self.stride[axis] + \
+      (self.size[axis] - 1) / 2
+
+  def compute_feature_coordinates(self, x, axis=None):
+    """
+    Computes the position of a feature given the center of a receptive field.
+
+    Args:
+      x: An array of input center coordinates with shape `(..., d)`, where `d`
+        is the number of dimensions of the coordinates.
+      axis: The dimensions for which to compute the feature coordinates.
+        If `None` (the default), compute the feature coordinates for all
+        dimensions.
+
+    Returns:
+      y: Coordinates of the features.
+
+    Raises:
+      ValueError: If the number of dimensions of the input center coordinates
+        does not match the number of elements in `axis`.
+    """
+    # Use all dimensions.
+    if axis is None:
+      axis = range(self.size.size)
+    # Ensure axis is a list because tuples have different indexing behavior.
+    axis = list(axis)
+    x = np.asarray(x)
+    if x.shape[-1] != len(axis):
+      raise ValueError("Dimensionality of the input center coordinates `x` "
+                       "(%d) does not match dimensionality of `axis` (%d)" %
+                       (x.shape[-1], len(axis)))
+    return (x + self.padding[axis] + (1 - self.size[axis]) / 2) / \
+      self.stride[axis]
+
+  def __iter__(self):
+    return iter(np.concatenate([self.size, self.stride, self.padding]))
+
+
+def compute_receptive_field_from_graph_def(graph_def, input_node, output_node,
+                                           stop_propagation=None):
+  """Computes receptive field (RF) parameters from a Graph or GraphDef object.
+
+  The algorithm stops the calculation of the receptive field whenever it
+  encounters an operation in the list `stop_propagation`. Stopping the
+  calculation early can be useful to calculate the receptive field of a
+  subgraph such as a single branch of the
+  [inception network](https://arxiv.org/abs/1512.00567).
 
   Args:
-    graph_def: GraphDef object.
-    input_node: Name of the input node from graph.
-    output_node: Name of the output node from graph.
+    graph_def: Graph or GraphDef object.
+    input_node: Name of the input node or Tensor object from graph.
+    output_node: Name of the output node or Tensor object from graph.
+    stop_propagation: List of operation or scope names for which to stop the
+      propagation of the receptive field.
 
   Returns:
     rf_size_x: Receptive field size of network in the horizontal direction, with
@@ -331,6 +424,18 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
       cannot be found. For network criterion alignment, see
       photos/vision/features/delf/g3doc/rf_computation.md
   """
+  # Convert a graph to graph_def if necessary.
+  if isinstance(graph_def, framework_ops.Graph):
+    graph_def = graph_def.as_graph_def()
+
+  # Convert tensors to names.
+  if isinstance(input_node, framework_ops.Tensor):
+    input_node = input_node.op.name
+  if isinstance(output_node, framework_ops.Tensor):
+    output_node = output_node.op.name
+
+  stop_propagation = stop_propagation or []
+
   # Computes order of computation for a given graph.
   name_to_order_node = graph_compute_order.get_compute_order(
       graph_def=graph_def)
@@ -422,6 +527,10 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
 
       # Loop over this node's inputs and potentially propagate information down.
       for inp_name in node.input:
+        # Stop the propagation of the receptive field.
+        if any(inp_name.startswith(stop) for stop in stop_propagation):
+          logging.vlog(3, "Skipping explicitly ignored node %s.", node.name)
+          continue
         logging.vlog(4, "inp_name = %s", inp_name)
         inp_node = name_to_order_node[inp_name].node
         logging.vlog(4, "inp_node = \n%s", inp_node)
@@ -480,6 +589,7 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
     raise ValueError("Output node was not found")
   if input_node not in rf_sizes_x:
     raise ValueError("Input node was not found")
-  return (rf_sizes_x[input_node], rf_sizes_y[input_node],
-          effective_strides_x[input_node], effective_strides_y[input_node],
-          effective_paddings_x[input_node], effective_paddings_y[input_node])
+  return ReceptiveField(
+    (rf_sizes_x[input_node], rf_sizes_y[input_node]),
+    (effective_strides_x[input_node], effective_strides_y[input_node]),
+    (effective_paddings_x[input_node], effective_paddings_y[input_node]))
diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
index 2771389250..8d7d5440f6 100644
--- a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
+++ b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
+import numpy as np
 
 
 def create_test_network_1():
@@ -150,6 +151,31 @@ def create_test_network_5():
   return g
 
 
+def create_test_network_6():
+  """Aligned network with dropout for test.
+
+  The graph is similar to create_test_network_1(), except that the right branch
+  has dropout normalization.
+
+  Returns:
+    g: Tensorflow graph object (Graph proto).
+  """
+  g = ops.Graph()
+  with g.as_default():
+    # An 8x8 test image.
+    x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image')
+    # Left branch.
+    l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID')
+    # Right branch.
+    l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]])
+    l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID')
+    l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID')
+    dropout = slim.dropout(l3)
+    # Addition.
+    nn.relu(l1 + dropout, name='output')
+  return g
+
+
 class RfUtilsTest(test.TestCase):
 
   def testComputeRFFromGraphDefAligned(self):
@@ -220,6 +246,36 @@ class RfUtilsTest(test.TestCase):
     self.assertEqual(effective_padding_x, 0)
     self.assertEqual(effective_padding_y, 0)
 
+  def testComputeRFFromGraphDefStopPropagation(self):
+    graph_def = create_test_network_6().as_graph_def()
+    input_node = 'input_image'
+    output_node = 'output'
+    # Compute the receptive field but stop the propagation for the random
+    # uniform variable of the dropout.
+    (receptive_field_x, receptive_field_y, effective_stride_x,
+     effective_stride_y, effective_padding_x, effective_padding_y) = (
+         receptive_field.compute_receptive_field_from_graph_def(
+             graph_def, input_node, output_node,
+             ['Dropout/dropout/random_uniform']))
+    self.assertEqual(receptive_field_x, 3)
+    self.assertEqual(receptive_field_y, 3)
+    self.assertEqual(effective_stride_x, 4)
+    self.assertEqual(effective_stride_y, 4)
+    self.assertEqual(effective_padding_x, 1)
+    self.assertEqual(effective_padding_y, 1)
+
+  def testComputeCoordinatesRoundtrip(self):
+    graph_def = create_test_network_1()
+    input_node = 'input_image'
+    output_node = 'output'
+    rf = receptive_field.compute_receptive_field_from_graph_def(
+      graph_def, input_node, output_node)
+
+    x = np.random.randint(0, 100, (50, 2))
+    y = rf.compute_feature_coordinates(x)
+    x2 = rf.compute_input_center_coordinates(y)
+
+    self.assertAllEqual(x, x2)
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
index 9a36bdc2f9..cd4d46aa07 100644
--- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
+++ b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 from tensorflow.contrib import stateless
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
@@ -79,6 +80,21 @@ class StatelessOpsTest(test.TestCase):
             for s1, v1 in values:
               self.assertEqual(s0 == s1, np.all(v0 == v1))
 
+  def testShapeType(self):
+    with self.test_session(use_gpu=True):
+      for shape_dtype in [dtypes.int32, dtypes.int64]:
+        seed_t = array_ops.placeholder(dtypes.int64, shape=[2])
+        seeds = [(x, y) for x in range(5) for y in range(5)] * 3
+        for stateless_op, _ in CASES:
+          for shape in (), (3,), (2, 5):
+            pure = stateless_op(constant_op.constant(shape, dtype=shape_dtype),
+                                seed=seed_t)
+            values = [(seed, pure.eval(feed_dict={seed_t: seed}))
+                      for seed in seeds]
+            for s0, v0 in values:
+              for s1, v1 in values:
+                self.assertEqual(s0 == s1, np.all(v0 == v1))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c4f880da9d..1c58aa3315 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -783,6 +783,7 @@ cc_library(
         "//tensorflow/core/kernels:dataset_ops",
         "//tensorflow/core/kernels:fake_quant_ops",
         "//tensorflow/core/kernels:function_ops",
+        "//tensorflow/core/kernels:histogram_op",
         "//tensorflow/core/kernels:image",
         "//tensorflow/core/kernels:io",
         "//tensorflow/core/kernels:linalg",
@@ -1943,6 +1944,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
 tf_cuda_library(
     name = "core_cpu_impl",
     srcs = [
+        "common_runtime/accumulate_n_optimizer.cc",
         "common_runtime/allocator_retry.cc",
         "common_runtime/bfc_allocator.cc",
         "common_runtime/build_graph_options.cc",
@@ -2178,6 +2180,7 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":protos_all_cc",
+        ":stream_executor",
         "//third_party/eigen3",
     ] + if_static([":gpu_runtime_impl"]),
 )
@@ -2673,6 +2676,22 @@ tf_cc_tests(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_runtime_tests",
+    size = "small",
+    srcs = ["common_runtime/mkl_cpu_allocator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":core",
+        ":core_cpu",
+        ":framework",
+        ":framework_internal",
+        ":test",
+        ":test_main",
+        ":testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_related_tests",
     size = "small",
@@ -2700,7 +2719,20 @@ tf_cc_test_mkl(
         "//tensorflow/cc:sendrecv_ops",
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
-    ],
+    ] + if_mkl([
+        "//tensorflow/core/kernels:mkl_aggregate_ops",
+        "//tensorflow/core/kernels:mkl_concat_op",
+        "//tensorflow/core/kernels:mkl_conv_op",
+        "//tensorflow/core/kernels:mkl_cwise_ops_common",
+        "//tensorflow/core/kernels:mkl_fused_batch_norm_op",
+        "//tensorflow/core/kernels:mkl_identity_op",
+        "//tensorflow/core/kernels:mkl_input_conversion_op",
+        "//tensorflow/core/kernels:mkl_lrn_op",
+        "//tensorflow/core/kernels:mkl_pooling_ops",
+        "//tensorflow/core/kernels:mkl_relu_op",
+        "//tensorflow/core/kernels:mkl_reshape_op",
+        "//tensorflow/core/kernels:mkl_tfconv_op",
+    ]),
 )
 
 tf_cc_tests_gpu(
diff --git a/tensorflow/core/common_runtime/accumulate_n_optimizer.cc b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
new file mode 100644
index 0000000000..81cd44870e
--- /dev/null
+++ b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
@@ -0,0 +1,191 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/graph/node_builder.h"
+
+
+namespace tensorflow {
+namespace {
+
+Tensor make_zeros(const DataType& dtype, const TensorShapeProto& shape) {
+  Tensor tensor(dtype, TensorShape(shape));
+
+  // Conveniently, all numeric data types have 0x0 == zero.  Otherwise we would
+  // need a giant switch statement here.
+  memset(const_cast(tensor.tensor_data().data()), 0,
+         tensor.tensor_data().size());
+
+  return tensor;
+}
+
+// Replaces occurrences of the "AccumulateNV2" stub operator with a graph of
+// lower-level ops. The graph is equivalent (modulo certain corner cases)
+// to the semantics of the original accumulate_n() Python op in math_ops.py.
+// Implementing the op with a rewrite allows this new variant of accumulate_n 
+// to be differentiable.
+//
+// The binary code that generates AccumulateNV2 stub ops is located in a
+// dynamic library built out of tensorflow/contrib/framework. Ideally, this
+// class would also be in contrib, but calls to REGISTER_OPTIMIZATION() from
+// third-party libraries aren't currently supported.
+class AccumulateNV2RemovePass : public GraphOptimizationPass {
+ public:
+
+  Status Run(const GraphOptimizationPassOptions& options) override {
+    // TODO(freiss.oss@gmail.com): Substantial shared code with
+    // ParallelConcatRemovePass::Run(). Consider refactoring if someone makes
+    // a third similar rewrite.
+    if (options.graph == nullptr) {
+      // TODO(apassos) returning OK feels weird here as we can't do anything
+      // without a graph, but some tests require this.
+      return Status::OK();
+    }
+
+    Graph* g = options.graph->get();
+    if (g == nullptr) {
+      return errors::Internal(
+          "AccumulateNV2 removal should happen before partitioning and a "
+          "graph should be available.");
+    }
+
+    // Build up a todo list of ops to replace, *then* modify the graph
+    gtl::InlinedVector matches;
+    for (Node* n : g->op_nodes()) {
+      if (n->type_string() == "AccumulateNV2") {
+        matches.push_back(n);
+      }
+    }
+    for (Node* n : matches) {
+      TF_RETURN_IF_ERROR(rewriteNode(n, g));
+    }
+    return Status::OK();
+  }
+
+  Status rewriteNode(Node* n, Graph* g) {
+    AttrSlice n_attrs = n->attrs();
+    auto base_make_node = [n, g, &n_attrs](const string& op,
+                                           const string& name) {
+      NodeBuilder node_builder(name, op);
+
+      // The pieces of AccumulateNV2 should all be on the same node.
+      node_builder.Device(n->requested_device());
+      string colo;
+      if (GetNodeAttr(n_attrs, kColocationAttrName, &colo).ok()) {
+        node_builder.Attr(kColocationAttrName, colo);
+      }
+      return node_builder;
+    };
+    auto make_node = [n, g, &n_attrs, &base_make_node](string op) {
+      return base_make_node(
+          op, g->NewName(strings::StrCat(n->name(), "/Internal")));
+    };
+
+    DataType dtype;
+    TF_RETURN_IF_ERROR(GetNodeAttr(n_attrs, "T", &dtype));
+    TensorShapeProto shape;
+    TF_RETURN_IF_ERROR(GetNodeAttr(n_attrs, "shape", &shape));
+
+    std::vector data_edges, control_edges;
+    for (const Edge* input_edge : n->in_edges()) {
+      if (input_edge->IsControlEdge()) {
+        control_edges.push_back(input_edge);
+      } else {
+        data_edges.push_back(input_edge);
+      }
+    }
+
+    // Create the following ops to replace the AccumulateNV2 placeholder:
+    Node* create_accumulator = nullptr;            // TemporaryVariable op
+    Node* initial_val = nullptr;                   // Const op
+    Node* initialize_accumulator = nullptr;        // Assign op
+    std::vector add_values_to_accumulator;  // AssignAdd ops
+    Node* clean_up_accumulator = nullptr;          // DestroyTemporaryVariable
+
+    const string accumulator_name =
+        strings::StrCat(n->name(), "/Internal/Accumulator");
+    TF_RETURN_IF_ERROR(make_node("TemporaryVariable")
+                           .Attr("shape", shape)
+                           .Attr("dtype", dtype)
+                           .Attr("var_name", accumulator_name)
+                           .Finalize(g, &create_accumulator));
+    TF_RETURN_IF_ERROR(make_node("Const")
+                           .Attr("value", make_zeros(dtype, shape))
+                           .Attr("dtype", dtype)
+                           .Finalize(g, &initial_val));
+    TF_RETURN_IF_ERROR(make_node("Assign")
+                           .Attr("T", dtype)
+                           .Input(create_accumulator)  // ref: Ref(T)
+                           .Input(initial_val)         // value: T
+                           .Finalize(g, &initialize_accumulator));
+    for (int i = 0; i < data_edges.size(); ++i) {
+      Node* assignAdd;
+      TF_RETURN_IF_ERROR(make_node("AssignAdd")
+                             .Attr("T", dtype)
+                             .Attr("use_locking", true)
+                             .Input(initialize_accumulator)  // ref: Ref(T)
+                             .Input(data_edges[i]->src(),
+                                    data_edges[i]->src_output())  // value: T
+                             .Finalize(g, &assignAdd));
+
+      add_values_to_accumulator.push_back(assignAdd);
+    }
+
+    // Note that we use the original placeholder op's name here
+    TF_RETURN_IF_ERROR(base_make_node("DestroyTemporaryVariable", n->name())
+                           .Attr("T", dtype)
+                           .Attr("var_name", accumulator_name)
+                           .Input(initialize_accumulator)
+                           .Finalize(g, &clean_up_accumulator));
+
+    // Add edges to the graph to ensure that operations occur in the right
+    // order:
+    // 1. Do anything that had a control edge to the AccumulateNV2 placeholder
+    // 2. Initialize accumulator
+    // 3. Add input values to accumulator (already handled by data edges
+    //    added above)
+    // 4. Reclaim the buffer that held the accumulator
+    // 5. Do anything that depended on the AccumulateNV2 placeholder
+    for (const Edge* control_edge : control_edges) {
+      g->AddControlEdge(control_edge->src(), initialize_accumulator);
+    }
+
+    for (Node* assign_add : add_values_to_accumulator) {
+      g->AddControlEdge(assign_add, clean_up_accumulator);
+    }
+
+    for (const Edge* out_edge : n->out_edges()) {
+      if (out_edge->IsControlEdge()) {
+        g->AddControlEdge(clean_up_accumulator, out_edge->dst());
+      } else {
+        g->AddEdge(clean_up_accumulator, 0, out_edge->dst(),
+                   out_edge->dst_input());
+      }
+    }
+
+    // Remove the original AccumulateNV2 placeholder op.
+    // This removal modifies the op and must happen after we have finished
+    // using its incoming/outgoing edge sets.
+    g->RemoveNode(n);
+
+    return Status::OK();
+  }
+};
+REGISTER_OPTIMIZATION(OptimizationPassRegistry::PRE_PLACEMENT, 0,
+                      AccumulateNV2RemovePass);
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index f16da10d7a..53e80b1ee3 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -21,9 +21,13 @@ limitations under the License.
 
 #ifdef INTEL_MKL
 
+#include 
+#include 
 #include 
 #include "tensorflow/core/common_runtime/bfc_allocator.h"
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/mem.h"
 
 #include "i_malloc.h"
@@ -46,10 +50,50 @@ class MklCPUAllocator : public Allocator {
  public:
   // Constructor and other standard functions
 
-  MklCPUAllocator() {
+  /// Environment variable that user can set to upper bound on memory allocation
+  static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
+
+  /// Default upper limit on allocator size - 64GB
+  static const size_t kDefaultMaxLimit = 64LL << 30;
+
+  MklCPUAllocator() { TF_CHECK_OK(Initialize()); }
+
+  ~MklCPUAllocator() override { delete allocator_; }
+
+  Status Initialize() {
     VLOG(2) << "MklCPUAllocator: In MklCPUAllocator";
-    allocator_ =
-        new BFCAllocator(new MklSubAllocator, kMaxMemSize, kAllowGrowth, kName);
+
+    // Set upper bound on memory allocation to physical RAM available on the
+    // CPU unless explicitly specified by user
+    uint64 max_mem_bytes = kDefaultMaxLimit;
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+    max_mem_bytes =
+        (uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
+#endif
+    char* user_mem_bytes = getenv(kMaxLimitStr);
+
+    if (user_mem_bytes != NULL) {
+      uint64 user_val = 0;
+      if (!strings::safe_strtou64(user_mem_bytes, &user_val)) {
+        return errors::InvalidArgument("Invalid memory limit (", user_mem_bytes,
+                                       ") specified for MKL allocator through ",
+                                       kMaxLimitStr);
+      }
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+      if (user_val > max_mem_bytes) {
+        LOG(WARNING) << "The user specifed a memory limit " << kMaxLimitStr
+                     << "=" << user_val
+                     << " greater than available physical memory: "
+                     << max_mem_bytes
+                     << ". This could significantly reduce performance!";
+      }
+#endif
+      max_mem_bytes = user_val;
+    }
+
+    VLOG(1) << "MklCPUAllocator: Setting max_mem_bytes: " << max_mem_bytes;
+    allocator_ = new BFCAllocator(new MklSubAllocator, max_mem_bytes,
+                                  kAllowGrowth, kName);
 
     // For redirecting all allocations from MKL to this allocator
     // From: http://software.intel.com/en-us/node/528565
@@ -57,9 +101,9 @@ class MklCPUAllocator : public Allocator {
     i_calloc = CallocHook;
     i_realloc = ReallocHook;
     i_free = FreeHook;
-  }
 
-  ~MklCPUAllocator() override { delete allocator_; }
+    return Status::OK();
+  }
 
   inline string Name() override { return kName; }
 
@@ -71,6 +115,8 @@ class MklCPUAllocator : public Allocator {
     allocator_->DeallocateRaw(ptr);
   }
 
+  void GetStats(AllocatorStats* stats) { return allocator_->GetStats(stats); }
+
  private:
   // Hooks provided by this allocator for memory allocation routines from MKL
 
@@ -96,11 +142,6 @@ class MklCPUAllocator : public Allocator {
     TF_CHECK_OK(s);  // way to assert with an error message
   }
 
-  // TODO(jbobba): We should ideally move this into CPUOptions in config.proto.
-  /// Memory limit - 64GB
-  static const size_t kMaxMemSize =
-      static_cast(64) * 1024 * 1024 * 1024;
-
   /// Do we allow growth in BFC Allocator
   static const bool kAllowGrowth = true;
 
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
new file mode 100644
index 0000000000..a67411cd2e
--- /dev/null
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
@@ -0,0 +1,53 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef INTEL_MKL
+
+#include "tensorflow/core/common_runtime/mkl_cpu_allocator.h"
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+TEST(MKLBFCAllocatorTest, TestMaxLimit) {
+  AllocatorStats stats;
+  setenv(MklCPUAllocator::kMaxLimitStr, "1000", 1);
+  MklCPUAllocator a;
+  TF_EXPECT_OK(a.Initialize());
+  a.GetStats(&stats);
+  EXPECT_EQ(stats.bytes_limit, 1000);
+
+  unsetenv(MklCPUAllocator::kMaxLimitStr);
+  TF_EXPECT_OK(a.Initialize());
+  a.GetStats(&stats);
+  uint64 max_mem_bytes = MklCPUAllocator::kDefaultMaxLimit;
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+  max_mem_bytes =
+      (uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
+#endif
+  EXPECT_EQ(stats.bytes_limit, max_mem_bytes);
+
+  setenv(MklCPUAllocator::kMaxLimitStr, "wrong-input", 1);
+  EXPECT_TRUE(errors::IsInvalidArgument(a.Initialize()));
+
+  setenv(MklCPUAllocator::kMaxLimitStr, "-20", 1);
+  EXPECT_TRUE(errors::IsInvalidArgument(a.Initialize()));
+}
+
+}  // namespace tensorflow
+
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 4796c3c00a..315c99d32b 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1020,6 +1020,29 @@ Status UnknownShape(shape_inference::InferenceContext* c) {
   return Status::OK();
 }
 
+template 
+Status ReductionShapeHelper(const Tensor* reduction_indices_t,
+                            const int32 input_rank,
+                            std::set& true_indices) {
+  auto reduction_indices = reduction_indices_t->flat();
+  for (int i = 0; i < reduction_indices_t->NumElements(); ++i) {
+    const T reduction_index = reduction_indices(i);
+    if (reduction_index < -input_rank || reduction_index >= input_rank) {
+      return errors::InvalidArgument("Invalid reduction dimension ",
+                                     reduction_index, " for input with ",
+                                     input_rank, " dimensions.");
+    }
+
+    auto wrapped_index = reduction_index;
+    if (wrapped_index < 0) {
+      wrapped_index += input_rank;
+    }
+
+    true_indices.insert(wrapped_index);
+  }
+  return Status::OK();
+}
+
 Status ReductionShape(InferenceContext* c) {
   ShapeHandle input = c->input(0);
 
@@ -1050,22 +1073,16 @@ Status ReductionShape(InferenceContext* c) {
   }
 
   const int32 input_rank = c->Rank(input);
-  std::set true_indices;
-  auto reduction_indices = reduction_indices_t->flat();
-  for (int i = 0; i < reduction_indices_t->NumElements(); ++i) {
-    int32 reduction_index = reduction_indices(i);
-    if (reduction_index < -input_rank || reduction_index >= input_rank) {
-      return errors::InvalidArgument("Invalid reduction dimension ",
-                                     reduction_index, " for input with ",
-                                     input_rank, " dimensions.");
-    }
-
-    int32 wrapped_index = reduction_index;
-    if (wrapped_index < 0) {
-      wrapped_index += input_rank;
-    }
-
-    true_indices.insert(wrapped_index);
+  std::set true_indices;
+  if (reduction_indices_t->dtype() == DataType::DT_INT32) {
+    TF_RETURN_IF_ERROR(ReductionShapeHelper(reduction_indices_t,
+                                                   input_rank, true_indices));
+  } else if (reduction_indices_t->dtype() == DataType::DT_INT64) {
+    TF_RETURN_IF_ERROR(ReductionShapeHelper(reduction_indices_t,
+                                                   input_rank, true_indices));
+  } else {
+    return errors::InvalidArgument(
+        "reduction_indices can only be int32 or int64");
   }
 
   std::vector dims;
@@ -1319,11 +1336,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) {
       Status s = c->Merge(prefix_indices, prefix_updates, &unused);
       if (!s.ok()) {
         return errors::InvalidArgument(
-            "The outer ", num_outer_dims,
-            " dimensions of indices.shape=", c->DebugString(indices_shape),
-            " must match the outer ", num_outer_dims,
-            " dimensions of updates.shape=", c->DebugString(updates_shape),
-            ": ", s.error_message());
+            "The outer ", num_outer_dims, " dimensions of indices.shape=",
+            c->DebugString(indices_shape), " must match the outer ",
+            num_outer_dims, " dimensions of updates.shape=",
+            c->DebugString(updates_shape), ": ", s.error_message());
       }
 
       ShapeHandle input_suffix;
diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto
index 53aa03108a..1fd2e50b51 100644
--- a/tensorflow/core/framework/node_def.proto
+++ b/tensorflow/core/framework/node_def.proto
@@ -35,7 +35,7 @@ message NodeDef {
   // CONSTRAINT ::= ("job:" JOB_NAME)
   //              | ("replica:" [1-9][0-9]*)
   //              | ("task:" [1-9][0-9]*)
-  //              | ( ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
+  //              | ("device:" ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
   //
   // Valid values for this string include:
   // * "/job:worker/replica:0/task:1/device:GPU:3"  (full specification)
diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h
index 61e722e57b..c31ab18cc1 100644
--- a/tensorflow/core/framework/register_types.h
+++ b/tensorflow/core/framework/register_types.h
@@ -87,7 +87,7 @@ limitations under the License.
 
 #elif defined(__ANDROID_TYPES_FULL__)
 
-// Only half, float, int32, int64, and quantized types are supported.
+// Only half, float, int32, int64, bool, and quantized types are supported.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
@@ -117,7 +117,7 @@ limitations under the License.
 
 #else  // defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__)
 
-// Only float and int32 are supported.
+// Only float, int32, and bool are supported.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
diff --git a/tensorflow/core/framework/rendezvous.cc b/tensorflow/core/framework/rendezvous.cc
index 90426defa0..a9e4c1cfb1 100644
--- a/tensorflow/core/framework/rendezvous.cc
+++ b/tensorflow/core/framework/rendezvous.cc
@@ -210,7 +210,7 @@ class LocalRendezvousImpl : public Rendezvous {
     ItemQueue* queue = &table_[key_hash];
     if (queue->empty() || !queue->front()->IsSendValue()) {
       // There is no message to pick up.
-      // Only recv-related fileds need to be filled.
+      // Only recv-related fields need to be filled.
       Item* item = new Item;
       item->waiter = std::move(done);
       item->recv_args = recv_args;
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 7c7f641265..c5dde722fa 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -639,7 +639,7 @@ class Graph {
   std::unordered_map device_names_map_;
 
   // All the while contexts owned by this graph, keyed by frame name,
-  // corresonding to all the while loops contained in this graph (including
+  // corresponding to all the while loops contained in this graph (including
   // nested loops). The stored contexts are usually accessed via
   // AddWhileContext() or Node::while_ctx(), but this manages the lifetime.
   std::map while_ctxs_;
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index f87a94a76a..f4c9073dee 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -543,7 +543,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string reason;
 
     // Substring that should be checked for in device name for CPU device.
-    const char* const kCPUDeviceSubStr = "cpu";
+    const char* const kCPUDeviceSubStr = "CPU";
 
     // If Op has been specifically assigned to a non-CPU device, then No.
     if (!n->assigned_device_name().empty() &&
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index a2b2f6530d..abc63e4f35 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -39,7 +39,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-const char kCPUDevice[] = "/job:a/replica:0/task:0/cpu:0";
+const char kCPUDevice[] = "/job:a/replica:0/task:0/device:CPU:0";
 const char kGPUDevice[] = "/job:a/replica:0/task:0/device:GPU:0";
 
 static void InitGraph(const string& s, Graph* graph,
diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc
index be52438747..172471e34b 100644
--- a/tensorflow/core/graph/testlib.cc
+++ b/tensorflow/core/graph/testlib.cc
@@ -480,6 +480,24 @@ Node* Conv2D(Graph* g, Node* in0, Node* in1) {
   return ret;
 }
 
+Node* Diag(Graph* g, Node* in, DataType type) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Diag")
+                  .Input(in)
+                  .Attr("T", type)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
+Node* DiagPart(Graph* g, Node* in, DataType type) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "DiagPart")
+                  .Input(in)
+                  .Attr("T", type)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
 void ToGraphDef(Graph* g, GraphDef* gdef) { g->ToGraphDef(gdef); }
 
 }  // end namespace graph
diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h
index a38809e6b4..06597778bb 100644
--- a/tensorflow/core/graph/testlib.h
+++ b/tensorflow/core/graph/testlib.h
@@ -199,6 +199,12 @@ Node* BiasAdd(Graph* g, Node* value, Node* bias);
 // Add a Conv2D node in "g".
 Node* Conv2D(Graph* g, Node* in0, Node* in1);
 
+// Add a Diag node in "g".
+Node* Diag(Graph* g, Node* in, DataType type);
+
+// Add a DiagPart node in "g".
+Node* DiagPart(Graph* g, Node* in, DataType type);
+
 }  // end namespace graph
 }  // end namespace test
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/model_pruner.cc b/tensorflow/core/grappler/optimizers/model_pruner.cc
index e087621c3b..b9df196f83 100644
--- a/tensorflow/core/grappler/optimizers/model_pruner.cc
+++ b/tensorflow/core/grappler/optimizers/model_pruner.cc
@@ -104,7 +104,7 @@ Status ModelPruner::Optimize(Cluster* cluster, const GrapplerItem& item,
     // - Don't remove nodes that receive reference values, as those can be
     //   converting references to non-references. It is important to preserve
     //   these non-references since the partitioner will avoid sending
-    //   non-references accross partitions more than once.
+    //   non-references across partitions more than once.
     if (!rewriter.DrivesControlDependency(node) &&
         !rewriter.IsDrivenByControlDependency(node) &&
         !rewriter.IsConnectedToFunction(node) &&
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index f5bfa60199..92a0dbd0ab 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2499,6 +2499,7 @@ cc_library(
         ":cross_op",
         ":cwise_op",
         ":fft_ops",
+        ":histogram_op",
         ":matmul_op",
         ":population_count_op",
         ":reduction_ops",
@@ -2635,6 +2636,24 @@ tf_kernel_library(
     deps = MATH_DEPS,
 )
 
+tf_cc_test(
+    name = "sequence_ops_test",
+    size = "small",
+    srcs = ["sequence_ops_test.cc"],
+    deps = [
+        ":ops_testutil",
+        ":ops_util",
+        ":sequence_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "cast_op_test",
     size = "small",
@@ -2893,6 +2912,24 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "diag_op_test",
+    size = "small",
+    srcs = ["diag_op_test.cc"],
+    deps = [
+        ":diag_op",
+        ":ops_testutil",
+        ":ops_util",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 # conv_grad_ops currently has to be built with conv_ops*.
 # TODO(josh11b, zhengxq): put these a separate libraries in ":nn" below once
 # conv_ops_gpu.h has be separated into its own library.
@@ -2993,6 +3030,7 @@ cc_library(
         ":in_topk_op",
         ":l2loss_op",
         ":lrn_op",
+        ":nth_element_op",
         ":relu_op",
         ":softmax_op",
         ":softplus_op",
@@ -3079,6 +3117,12 @@ tf_kernel_library(
     deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
+tf_kernel_library(
+    name = "nth_element_op",
+    prefix = "nth_element_op",
+    deps = NN_DEPS,
+)
+
 tf_kernel_library(
     name = "xent_op",
     prefix = "xent_op",
@@ -3096,6 +3140,17 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "histogram_op",
+    prefix = "histogram_op",
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//third_party/eigen3",
+    ] + if_cuda(["@cub_archive//:cub"]),
+)
+
 tf_kernel_library(
     name = "l2loss_op",
     prefix = "l2loss_op",
diff --git a/tensorflow/core/kernels/batchtospace_op.cc b/tensorflow/core/kernels/batchtospace_op.cc
index 99b5d3daaa..c1c0d6d329 100644
--- a/tensorflow/core/kernels/batchtospace_op.cc
+++ b/tensorflow/core/kernels/batchtospace_op.cc
@@ -249,40 +249,34 @@ class BatchToSpaceOp : public OpKernel {
   Tensor block_shape_;
 };
 
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")                     \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tblock_shape") \
-                              .TypeConstraint("Tcrops")       \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("crops"),                  \
-                          BatchToSpaceNDOp);           \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")                       \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tidx")         \
-                              .HostMemory("crops"),                  \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")           \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("crops"),        \
+                          BatchToSpaceNDOp); \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")             \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("crops"),        \
                           BatchToSpaceOp);
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER);
 #undef REGISTER
 
 #if GOOGLE_CUDA
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")                     \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tblock_shape") \
-                              .TypeConstraint("Tcrops")       \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("crops"),                  \
-                          BatchToSpaceNDOp);           \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")                       \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tidx")         \
-                              .HostMemory("crops"),                  \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")           \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("crops"),        \
+                          BatchToSpaceNDOp); \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")             \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("crops"),        \
                           BatchToSpaceOp);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index 6e10b53cf7..9a00a091bd 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -394,7 +394,7 @@ __global__ void SwapDimension1And2InTensor3SmallDim(const T* input,
     int output_block_idx = SmallDim2 ? block_offset : block_offset * small_dim;
     int output_block_origin_idx = output_block_offset + output_block_idx;
 
-    // Store the tranposed memory region in shared memory to device.
+    // Store the transposed memory region in shared memory to device.
     if (x < tile_height) {
       for (int y = 0; y < small_dim; y++) {
         int output_idx = output_block_origin_idx + x +
diff --git a/tensorflow/core/kernels/crop_and_resize_op_test.cc b/tensorflow/core/kernels/crop_and_resize_op_test.cc
index 22c659b587..a35e1b0788 100644
--- a/tensorflow/core/kernels/crop_and_resize_op_test.cc
+++ b/tensorflow/core/kernels/crop_and_resize_op_test.cc
@@ -61,8 +61,12 @@ class CropAndResizeOpTest : public OpsTestBase {
 
 REGISTER_TEST(float)
 REGISTER_TEST(double)
-REGISTER_TEST(int8)
 REGISTER_TEST(uint8)
+REGISTER_TEST(uint16)
+REGISTER_TEST(int8)
+REGISTER_TEST(int16)
+REGISTER_TEST(int32)
+REGISTER_TEST(int64)
 
 #undef REGISTER_TEST
 
diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h
index a906113466..a431889409 100644
--- a/tensorflow/core/kernels/dataset.h
+++ b/tensorflow/core/kernels/dataset.h
@@ -412,7 +412,7 @@ class DatasetIterator : public IteratorBase {
     // Owns one reference on the shared dataset resource.
     const DatasetType* dataset;
 
-    // Identifies the sequence of iterators leading up to to this iterator.
+    // Identifies the sequence of iterators leading up to this iterator.
     const string prefix;
   };
 
diff --git a/tensorflow/core/kernels/diag_op.cc b/tensorflow/core/kernels/diag_op.cc
index c800859d90..be862b82f1 100644
--- a/tensorflow/core/kernels/diag_op.cc
+++ b/tensorflow/core/kernels/diag_op.cc
@@ -14,65 +14,32 @@ limitations under the License.
 ==============================================================================*/
 
 // See docs in ../ops/array_ops.cc
+
+#define EIGEN_USE_THREADS
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#endif  // GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/diag_op.h"
+
+#include 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
-namespace {
-template 
-class DiagonalGenerator {
- public:
-  explicit DiagonalGenerator(const Tensor& diagonal) : diagonal_(diagonal) {
-    static_assert(DoubleNumDims == 2 * NumDims,
-                  "The second size must be the double of the first size.");
-    CHECK_EQ(diagonal.dims(), NumDims);
-  }
-  T operator()(
-      const Eigen::array& coordinates) const {
-    Eigen::array index;
-    for (size_t i = 0; i < NumDims; ++i) {
-      if (coordinates[i] != coordinates[NumDims + i]) {
-        return T(0);
-      }
-      index[i] = coordinates[i];
-    }
-    return diagonal_.tensor()(index);
-  }
 
- private:
-  Tensor diagonal_;
-};
-
-template 
-class DiagonalExtractor {
- public:
-  explicit DiagonalExtractor(const Tensor& tensor) : tensor_(tensor) {
-    CHECK_EQ(tensor.dims(), 2 * NumDims);
-  }
-  T operator()(const Eigen::array& coordinates) const {
-    Eigen::array index;
-    for (size_t j = 0; j < NumDims; ++j){
-      index[j] = coordinates[j];
-    }
-    for (size_t j = NumDims; j < 2 * NumDims; ++j){
-      index[j] = index[j - NumDims];
-    }
-    return tensor_.tensor()(index);
-  }
-
- private:
-  Tensor tensor_;
-};
-  
-}  // namespace
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
 
 // Generate the diagonal tensor with the diagonal set to the input tensor.
-// It only allows up to rank 3 input tensor, so the output tensor is up to
-// rank 6.
-template 
+template 
 class DiagOp : public OpKernel {
  public:
   explicit DiagOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -80,9 +47,8 @@ class DiagOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& diagonal = context->input(0);
     const int num_dims = diagonal.dims();
-    OP_REQUIRES(context, 1 <= num_dims && num_dims <= 3,
-                errors::InvalidArgument("Expected 1 <= dims <= 3, got shape ",
-                                        diagonal.shape().DebugString()));
+    OP_REQUIRES(context, 0 != num_dims, errors::InvalidArgument(
+        "Input must be at least rank 1, got 0"));
     TensorShape out_shape;
     for (int i = 0; i < num_dims; ++i) {
       out_shape.AddDim(diagonal.dim_size(i));
@@ -93,45 +59,17 @@ class DiagOp : public OpKernel {
     Tensor* output_tensor = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, out_shape, &output_tensor));
-    switch (num_dims) {
-      case 1:
-        output_tensor->tensor() = output_tensor->tensor().generate(
-            DiagonalGenerator(diagonal));
-        break;
-      case 2:
-        output_tensor->tensor() = output_tensor->tensor().generate(
-            DiagonalGenerator(diagonal));
-        break;
-      case 3:
-        output_tensor->tensor() = output_tensor->tensor().generate(
-            DiagonalGenerator(diagonal));
-        break;
-      default:
-        context->SetStatus(errors::Unimplemented(
-            "Diagonal of rank ", num_dims, " tensor is not supported yet."));
-        return;
-    }
+    functor::DiagFunctor diagFunc;
+    Status s = diagFunc(context,
+                        diagonal.NumElements(),
+                        diagonal.flat().data(),
+                        output_tensor->flat().data());
+    OP_REQUIRES_OK(context, s);
   }
 };
 
-#define REGISTER_DIAGOP(T) \
-  REGISTER_KERNEL_BUILDER( \
-      Name("Diag").Device(DEVICE_CPU).TypeConstraint("T"), DiagOp)
-
-REGISTER_DIAGOP(double);
-REGISTER_DIAGOP(float);
-REGISTER_DIAGOP(int32);
-REGISTER_DIAGOP(int64);
-REGISTER_DIAGOP(complex64);
-REGISTER_DIAGOP(complex128);
-
-#undef REGISTER_DIAGOP
-
-
-// Generate the diagonal tensor with the diagonal set to the input tensor.
-// It only allows rank 2, 4, or 6 input tensor, so the output tensor is 
-// rank 1, 2, or 3.
-template 
+// Extract the diagonal tensor with the diagonal set to the input tensor.
+template 
 class DiagPartOp : public OpKernel {
  public:
   explicit DiagPartOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -140,9 +78,9 @@ class DiagPartOp : public OpKernel {
     const Tensor& tensor = context->input(0);
     const int num_dims = tensor.dims();
     const int out_dims = num_dims / 2;
-    OP_REQUIRES(context, 2 == num_dims || 4 == num_dims || 6 == num_dims, 
-                errors::InvalidArgument("The rank of the tensor should be 2, \
-                                         4, or 6, got shape ",
+    OP_REQUIRES(context, 0 == num_dims % 2,
+                errors::InvalidArgument("The rank of the tensor should be \
+                                         even and positive, got shape ",
                                         tensor.shape().DebugString()));
     for (int i = 0; i < out_dims; i++){
       OP_REQUIRES(context, tensor.dim_size(i) == tensor.dim_size(i + out_dims),
@@ -160,39 +98,158 @@ class DiagPartOp : public OpKernel {
     Tensor* output = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, out_shape, &output));
+    functor::DiagPartFunctor diagPartFunc;
+    Status s = diagPartFunc(context,
+                            out_shape.num_elements(),
+                            tensor.flat().data(),
+                            output->flat().data());
+    OP_REQUIRES_OK(context, s);
+  }
+};
 
-    switch (num_dims) {
-      case 2:
-        output->tensor() = output->tensor().generate(
-          DiagonalExtractor(tensor));
-        break; 
-      case 4:
-        output->tensor() = output->tensor().generate(
-          DiagonalExtractor(tensor));
-        break;
-      case 6:
-        output->tensor() = output->tensor().generate(
-          DiagonalExtractor(tensor));
-        break;      
-      default:
-        context->SetStatus(errors::Unimplemented(
-          "Diagonal of rank ", num_dims, " tensor is not supported yet."));
-        return;
-    }
+// Implementation of the functor specialization for CPU.
+// 
+// According to the diagonal definition,
+// `output[i1,..., ik, i1,..., ik] = input[i1,..., ik]`,
+//
+// Let the rank of input is [s1,..., sk], then any offset of input's
+// pointer can be represent by coordinate [i1,..., ik],
+// where `index = i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik`
+//
+// Let new_index is the offset of output's pointer with coordinate 
+// [i1,..., ik, i1,..., ik], then we have
+// `new_index = i1*(s2*...sk*s1*...*sk) + i2*(s3*...*sk*s1*...*sk) +... + \
+//              ik*(s1*...*sk) + i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik
+//            = (i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik) * (1 + s1*...*sk)
+//            = index * (1 + s1*...*sk)
+//
+// Let `size = s1*...*sk`, we finally have `new_index = index * (1 + size)`,
+// which is the transfer function we use below.
+// This trick make our implementations clear and easy to be parallel.
+namespace functor {
+template 
+struct DiagFunctor {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // This subprocess is responsible for writing values in index range
+    // [start*size, limit*size)
+    auto subDiag = [in, out, size](int64 start, int64 limit) {
+      std::fill(out + size * start, out + size * limit, T());
+      for (int64 index = start; index < limit; ++index) {
+        out[(1 + size) * index] = in[index];
+      }
+    };
+
+    // Here, 5 is a empirical factor of cost_per_unit.
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers, size,
+        5 * size, subDiag);
+    return Status::OK();
+  }
+};
+
+template 
+struct DiagPartFunctor {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // This subprocess is responsible for extracting values in index range
+    // [start, limit)
+    auto subDiagPart = [in, out, size](int64 start, int64 limit) {
+      for (int64 index = start; index < limit; ++index) {
+        out[index] = in[(1 + size) * index];
+      }
+    };
+
+    // Here, 5 is a empirical factor of cost_per_unit.
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers, size,
+        5, subDiagPart);
+    return Status::OK();
   }
 };
+}  // namespace functor
 
-#define REGISTER_DIAGPARTOP(T) \
-  REGISTER_KERNEL_BUILDER( \
-      Name("DiagPart").Device(DEVICE_CPU).TypeConstraint("T"), DiagPartOp)
 
-REGISTER_DIAGPARTOP(double);
-REGISTER_DIAGPARTOP(float);
-REGISTER_DIAGPARTOP(int32);
-REGISTER_DIAGPARTOP(int64);
-REGISTER_DIAGPARTOP(complex64);
-REGISTER_DIAGPARTOP(complex128);
+// Register the CPU kernels.
+#define REGISTER_DIAGOP(T)                                    \
+  REGISTER_KERNEL_BUILDER(                                    \
+      Name("Diag").Device(DEVICE_CPU).TypeConstraint("T"), \
+      DiagOp)
 
+TF_CALL_double(REGISTER_DIAGOP);
+TF_CALL_float(REGISTER_DIAGOP);
+TF_CALL_int32(REGISTER_DIAGOP);
+TF_CALL_int64(REGISTER_DIAGOP);
+TF_CALL_complex64(REGISTER_DIAGOP);
+TF_CALL_complex128(REGISTER_DIAGOP);
+#undef REGISTER_DIAGOP
+
+#define REGISTER_DIAGPARTOP(T)                                    \
+  REGISTER_KERNEL_BUILDER(                                        \
+      Name("DiagPart").Device(DEVICE_CPU).TypeConstraint("T"), \
+      DiagPartOp)
+
+TF_CALL_double(REGISTER_DIAGPARTOP);
+TF_CALL_float(REGISTER_DIAGPARTOP);
+TF_CALL_int32(REGISTER_DIAGPARTOP);
+TF_CALL_int64(REGISTER_DIAGPARTOP);
+TF_CALL_complex64(REGISTER_DIAGPARTOP);
+TF_CALL_complex128(REGISTER_DIAGPARTOP);
 #undef REGISTER_DIAGPARTOP
-  
+
+// Register the GPU kernels.
+#ifdef GOOGLE_CUDA
+
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+extern template struct DiagFunctor;
+extern template struct DiagFunctor;
+extern template struct DiagFunctor;
+extern template struct DiagFunctor;
+extern template struct DiagFunctor;
+extern template struct DiagFunctor;
+}  // namespace functor
+
+#define REGISTER_DIAGOP_GPU(T)                                \
+  REGISTER_KERNEL_BUILDER(                                    \
+      Name("Diag").Device(DEVICE_GPU).TypeConstraint("T"), \
+      DiagOp)
+
+TF_CALL_double(REGISTER_DIAGOP_GPU);
+TF_CALL_float(REGISTER_DIAGOP_GPU);
+TF_CALL_int32(REGISTER_DIAGOP_GPU);
+TF_CALL_int64(REGISTER_DIAGOP_GPU);
+TF_CALL_complex64(REGISTER_DIAGOP_GPU);
+TF_CALL_complex128(REGISTER_DIAGOP_GPU);
+#undef REGISTER_DIAGOP_GPU
+
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+extern template struct DiagPartFunctor;
+extern template struct DiagPartFunctor;
+extern template struct DiagPartFunctor;
+extern template struct DiagPartFunctor;
+extern template struct DiagPartFunctor;
+extern template struct DiagPartFunctor;
+}  // namespace functor
+
+#define REGISTER_DIAGPARTOP_GPU(T)                                \
+  REGISTER_KERNEL_BUILDER(                                        \
+      Name("DiagPart").Device(DEVICE_GPU).TypeConstraint("T"), \
+      DiagPartOp)
+
+TF_CALL_double(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_float(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_int32(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_int64(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_complex64(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_complex128(REGISTER_DIAGPARTOP_GPU);
+#undef REGISTER_DIAGPARTOP_GPU
+
+#endif  // GOOGLE_CUDA
+
+
 }  // namespace tensorflow
+
diff --git a/tensorflow/core/kernels/diag_op.h b/tensorflow/core/kernels/diag_op.h
new file mode 100644
index 0000000000..c6ca6a2047
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op.h
@@ -0,0 +1,43 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
+#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template 
+struct DiagFunctor {
+  Status operator() (OpKernelContext* context, const int64 size,
+                     const T* in, T* out);
+};
+
+template 
+struct DiagPartFunctor {
+  Status operator() (OpKernelContext* context, const int64 size,
+                     const T* in, T* out);
+};
+
+}  // namespace functor
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc
new file mode 100644
index 0000000000..684f00ea61
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc
@@ -0,0 +1,139 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include 
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+#include "tensorflow/core/kernels/diag_op.h"
+
+namespace tensorflow {
+namespace functor {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+template 
+__global__ void DiagCudaKernel(const int num_threads,
+                               const int64 size,
+                               const T* in,
+                               T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    // Fill the diagonal elements or set to zero in other place. 
+    if (index % (1 + size) == 0) {
+      out[index] = in[index / (1 + size)];
+    } else {
+      out[index] = T(0);
+    }
+  }
+}
+
+template 
+struct DiagFunctor {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // Empty tensor couldn't launch the kernel.
+    if (size == 0) {
+      return Status::OK();
+    }
+
+    // CudaLaunchConfig uses an int for virtual_thread_count,
+    // so this may overflow for `size*size` in extreme cases,
+    // here is checking the multiplication overflow for integer.
+    if (size && (int(size * size) / size) != size) {
+      return errors::Internal(
+          "DiagOp got input size too large.");
+    }
+    int virtual_thread_count = int(size * size);
+
+    // Launch the GPU kernel.
+    const GPUDevice& device = context->eigen_device();
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(
+        virtual_thread_count, device);
+    DiagCudaKernel<<>>(
+        diag_config.virtual_thread_count, size, in, out);
+
+    auto err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+    return Status::OK();
+  }
+};
+
+template struct DiagFunctor;
+template struct DiagFunctor;
+template struct DiagFunctor;
+template struct DiagFunctor;
+template struct DiagFunctor;
+template struct DiagFunctor;
+
+
+template 
+__global__ void DiagPartCudaKernel(const int num_threads,
+                                   const int64 size,
+                                   const T* in,
+                                   T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    out[index] = in[(1 + size) * index];
+  }
+}
+
+template 
+struct DiagPartFunctor {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // Empty tensor couldn't launch the kernel.
+    if (size == 0) {
+      return Status::OK();
+    }
+    const GPUDevice& device = context->eigen_device();
+
+    // Extract the diagonal elements.
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device);
+    DiagPartCudaKernel<<>>(
+        diag_config.virtual_thread_count, size, in, out);
+
+    auto err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagPartOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+    return Status::OK();
+  }
+};
+
+template struct DiagPartFunctor;
+template struct DiagPartFunctor;
+template struct DiagPartFunctor;
+template struct DiagPartFunctor;
+template struct DiagPartFunctor;
+template struct DiagPartFunctor;
+
+}  // end namespace functor
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/diag_op_test.cc b/tensorflow/core/kernels/diag_op_test.cc
new file mode 100644
index 0000000000..2d1417854c
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op_test.cc
@@ -0,0 +1,54 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+template 
+static Graph* Diag(int n, DataType type) {
+  Graph* g = new Graph(OpRegistry::Global());
+  Tensor in(type, TensorShape({n}));
+  in.flat().setRandom();
+  Node* out = test::graph::Diag(g, test::graph::Constant(g, in), type);
+  test::graph::DiagPart(g, out, type);
+  return g;
+}
+
+#define BM_DiagDev(N, T, TFTYPE, DEVICE)                           \
+  static void BM_Diag##_##N##_##TFTYPE##_##DEVICE(int iters) {   \
+    testing::UseRealTime();                                     \
+    testing::ItemsProcessed(static_cast(iters) * N * N); \
+    test::Benchmark(#DEVICE, Diag(N, TFTYPE)).Run(iters);    \
+  }                                                             \
+  BENCHMARK(BM_Diag##_##N##_##TFTYPE##_##DEVICE);
+
+#define BM_Diag(N)                                       \
+  BM_DiagDev(N, int, DT_INT32, cpu);                     \
+  BM_DiagDev(N, float, DT_FLOAT, cpu);                   \
+  BM_DiagDev(N, std::complex, DT_COMPLEX64, cpu); \
+  BM_DiagDev(N, int, DT_INT32, gpu);                     \
+  BM_DiagDev(N, float, DT_FLOAT, gpu);                   \
+  BM_DiagDev(N, std::complex, DT_COMPLEX64, gpu);
+
+BM_Diag(16);
+BM_Diag(128);
+BM_Diag(512);
+
+}  // end namespace tensorflow
+
diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc
new file mode 100644
index 0000000000..4e035286f6
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op.cc
@@ -0,0 +1,147 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/math_ops.cc.
+
+#define EIGEN_USE_THREADS
+
+#include "tensorflow/core/kernels/histogram_op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace functor {
+
+template 
+struct HistogramFixedWidthFunctor {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes::ConstTensor& values,
+                        const typename TTypes::ConstTensor& value_range,
+                        int32 nbins, typename TTypes::Tensor& out) {
+    const CPUDevice& d = context->eigen_device();
+
+    Tensor index_to_bin_tensor;
+
+    TF_RETURN_IF_ERROR(context->forward_input_or_allocate_temp(
+        {0}, DataTypeToEnum::value, TensorShape({values.size()}),
+        &index_to_bin_tensor));
+    auto index_to_bin = index_to_bin_tensor.flat();
+
+    const double step = static_cast(value_range(1) - value_range(0)) /
+                        static_cast(nbins);
+
+    // The calculation is done by finding the slot of each value in `values`.
+    // With [a, b]:
+    //   step = (b - a) / nbins
+    //   (x - a) / step
+    // , then the entries are mapped to output.
+    index_to_bin.device(d) =
+        ((values.cwiseMax(value_range(0)) - values.constant(value_range(0)))
+             .template cast() /
+         step)
+            .template cast()
+            .cwiseMin(nbins - 1);
+
+    out.setZero();
+    for (int32 i = 0; i < index_to_bin.size(); i++) {
+      out(index_to_bin(i)) += Tout(1);
+    }
+    return Status::OK();
+  }
+};
+
+}  // namespace functor
+
+template 
+class HistogramFixedWidthOp : public OpKernel {
+ public:
+  explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& values_tensor = ctx->input(0);
+    const Tensor& value_range_tensor = ctx->input(1);
+    const Tensor& nbins_tensor = ctx->input(2);
+
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()),
+                errors::InvalidArgument("value_range should be a vector."));
+    OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2),
+                errors::InvalidArgument(
+                    "value_range should be a vector of 2 elements."));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(nbins_tensor.shape()),
+                errors::InvalidArgument("nbins should be a scalar."));
+
+    const auto values = values_tensor.flat();
+    const auto value_range = value_range_tensor.flat();
+    const auto nbins = nbins_tensor.scalar()();
+
+    OP_REQUIRES(
+        ctx, (value_range(0) < value_range(1)),
+        errors::InvalidArgument("value_range should satisfy value_range[0] < "
+                                "value_range[1], but got '[",
+                                value_range(0), ", ", value_range(1), "]'"));
+    OP_REQUIRES(
+        ctx, (nbins > 0),
+        errors::InvalidArgument("nbins should be a positive number, but got '",
+                                nbins, "'"));
+
+    Tensor* out_tensor;
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output(0, TensorShape({nbins}), &out_tensor));
+    auto out = out_tensor->flat();
+
+    OP_REQUIRES_OK(
+        ctx, functor::HistogramFixedWidthFunctor::Compute(
+                 ctx, values, value_range, nbins, out));
+  }
+};
+
+#define REGISTER_KERNELS(type)                                           \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")                    \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint("T")                 \
+                              .TypeConstraint("dtype"),           \
+                          HistogramFixedWidthOp) \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")                    \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint("T")                 \
+                              .TypeConstraint("dtype"),           \
+                          HistogramFixedWidthOp)
+
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS);
+#undef REGISTER_KERNELS
+
+#if GOOGLE_CUDA
+#define REGISTER_KERNELS(type)                                 \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")          \
+                              .Device(DEVICE_GPU)              \
+                              .HostMemory("value_range")       \
+                              .HostMemory("nbins")             \
+                              .TypeConstraint("T")       \
+                              .TypeConstraint("dtype"), \
+                          HistogramFixedWidthOp)
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_KERNELS);
+#undef REGISTER_KERNELS
+
+#endif  // GOOGLE_CUDA
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/histogram_op.h b/tensorflow/core/kernels/histogram_op.h
new file mode 100644
index 0000000000..1b253f7fed
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op.h
@@ -0,0 +1,38 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_HISTOGRAM_OP_H_
+#define TENSORFLOW_HISTOGRAM_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace tensorflow {
+namespace functor {
+
+template 
+struct HistogramFixedWidthFunctor {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes::ConstTensor& values,
+                        const typename TTypes::ConstTensor& value_range,
+                        int32 nbins, typename TTypes::Tensor& out);
+};
+
+}  // end namespace functor
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_HISTOGRAM_OP_H_
diff --git a/tensorflow/core/kernels/histogram_op_gpu.cu.cc b/tensorflow/core/kernels/histogram_op_gpu.cu.cc
new file mode 100644
index 0000000000..c2bb958be8
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op_gpu.cu.cc
@@ -0,0 +1,125 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/histogram_op.h"
+#include "external/cub_archive/cub/device/device_histogram.cuh"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace functor {
+
+// TODO(yongtang) int64 of atomicAdd is not supported yet.
+template 
+struct HistogramFixedWidthFunctor {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes::ConstTensor& values,
+                        const typename TTypes::ConstTensor& value_range,
+                        int32 nbins, typename TTypes::Tensor& out) {
+    tensorflow::AllocatorAttributes pinned_allocator;
+    pinned_allocator.set_on_host(true);
+    pinned_allocator.set_gpu_compatible(true);
+
+    Tensor levels_tensor;
+    TF_RETURN_IF_ERROR(context->allocate_temp(
+        DataTypeToEnum::value, TensorShape({nbins + 1}), &levels_tensor,
+        pinned_allocator));
+    auto levels = levels_tensor.flat();
+
+    const double step = static_cast(value_range(1) - value_range(0)) /
+                        static_cast(nbins);
+    levels(0) = std::numeric_limits::lowest();
+    for (int i = 1; i < nbins; i++) {
+      levels(i) =
+          static_cast(static_cast(value_range(0)) + step * i);
+    }
+    levels(nbins) = std::numeric_limits::max();
+
+    size_t temp_storage_bytes = 0;
+    const T* d_samples = values.data();
+    Tout* d_histogram = out.data();
+    int num_levels = levels.size();
+    T* d_levels = levels.data();
+    int num_samples = values.size();
+    const cudaStream_t& stream = GetCudaStream(context);
+
+    // The first HistogramRange is to obtain the temp storage size required
+    // with d_temp_storage = NULL passed to the call.
+    auto err = cub::DeviceHistogram::HistogramRange(
+        /* d_temp_storage */ NULL,
+        /* temp_storage_bytes */ temp_storage_bytes,
+        /* d_samples */ d_samples,
+        /* d_histogram */ d_histogram,
+        /* num_levels */ num_levels,
+        /* d_levels */ d_levels,
+        /* num_samples */ num_samples,
+        /* stream */ stream);
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch HistogramRange to get temp storage: ",
+          cudaGetErrorString(err), ".");
+    }
+
+    Tensor temp_storage;
+    TF_RETURN_IF_ERROR(context->allocate_temp(
+        DataTypeToEnum::value,
+        TensorShape({static_cast(temp_storage_bytes)}), &temp_storage));
+
+    void* d_temp_storage = temp_storage.flat().data();
+
+    // The second HistogramRange is to actual run with d_temp_storage
+    // allocated with temp_storage_bytes.
+    err = cub::DeviceHistogram::HistogramRange(
+        /* d_temp_storage */ d_temp_storage,
+        /* temp_storage_bytes */ temp_storage_bytes,
+        /* d_samples */ d_samples,
+        /* d_histogram */ d_histogram,
+        /* num_levels */ num_levels,
+        /* d_levels */ d_levels,
+        /* num_samples */ num_samples,
+        /* stream */ stream);
+    if (err != cudaSuccess) {
+      return errors::Internal("Could not launch HistogramRange: ",
+                              cudaGetErrorString(err), ".");
+    }
+
+    return Status::OK();
+  }
+};
+
+}  // end namespace functor
+
+#define REGISTER_GPU_SPEC(type) \
+  template struct functor::HistogramFixedWidthFunctor;
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_SPEC);
+#undef REGISTER_GPU_SPEC
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/listdiff_op.cc b/tensorflow/core/kernels/listdiff_op.cc
index d303bdd560..d28a2729d4 100644
--- a/tensorflow/core/kernels/listdiff_op.cc
+++ b/tensorflow/core/kernels/listdiff_op.cc
@@ -24,12 +24,13 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
-template 
+template 
 class ListDiffOp : public OpKernel {
  public:
   explicit ListDiffOp(OpKernelConstruction* context) : OpKernel(context) {
     const DataType dt = DataTypeToEnum::v();
-    OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, DT_INT32}));
+    const DataType dtidx = DataTypeToEnum::v();
+    OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, dtidx}));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -72,9 +73,9 @@ class ListDiffOp : public OpKernel {
 
     Tensor* indices = nullptr;
     OP_REQUIRES_OK(context, context->allocate_output(1, {out_size}, &indices));
-    auto Tindices = indices->vec();
+    auto Tindices = indices->vec();
 
-    for (int i = 0, p = 0; i < static_cast(x_size); ++i) {
+    for (Tidx i = 0, p = 0; i < static_cast(x_size); ++i) {
       if (y_set.count(Tx(i)) == 0) {
         OP_REQUIRES(context, p < out_size,
                     errors::InvalidArgument(
@@ -95,7 +96,12 @@ class ListDiffOp : public OpKernel {
                               .Device(DEVICE_CPU)                \
                               .TypeConstraint("T")         \
                               .TypeConstraint("out_idx"), \
-                          ListDiffOp)
+                          ListDiffOp)               \
+  REGISTER_KERNEL_BUILDER(Name("ListDiff")                       \
+                              .Device(DEVICE_CPU)                \
+                              .TypeConstraint("T")         \
+                              .TypeConstraint("out_idx"), \
+                          ListDiffOp)
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_LISTDIFF);
 REGISTER_LISTDIFF(string);
diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc
index 0168b57d35..7b5a464b72 100644
--- a/tensorflow/core/kernels/map_stage_op.cc
+++ b/tensorflow/core/kernels/map_stage_op.cc
@@ -111,15 +111,21 @@ class StagingMap : public ResourceBase {
   void notify_inserters_if_bounded(std::unique_lock* lock) {
     if (has_capacity() || has_memory_limit()) {
       lock->unlock();
-      full_.notify_one();
+      // Notify all inserters. The removal of an element
+      // may make memory available for many inserters
+      // to insert new elements
+      full_.notify_all();
     }
   }
 
-  // Notify any removers waiting to extract values
+  // Notify all removers waiting to extract values
   // that data is now available
   void notify_removers(std::unique_lock* lock) {
     lock->unlock();
-    not_empty_.notify_one();
+    // Notify all removers. This is because they are
+    // waiting for specific keys to appear in the map
+    // so we don't know which one to wake up.
+    not_empty_.notify_all();
   }
 
   bool has_capacity() const { return capacity_ > 0; }
diff --git a/tensorflow/core/kernels/mirror_pad_op.cc b/tensorflow/core/kernels/mirror_pad_op.cc
index e3643f9447..fbdeaf43eb 100644
--- a/tensorflow/core/kernels/mirror_pad_op.cc
+++ b/tensorflow/core/kernels/mirror_pad_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/mirror_pad_op.h"
-
 #include 
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -35,7 +35,7 @@ limitations under the License.
 
 namespace tensorflow {
 
-template 
+template 
 class MirrorPadOp : public OpKernel {
  public:
   explicit MirrorPadOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -82,10 +82,10 @@ class MirrorPadOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes::ConstMatrix paddings = in1.matrix();
+    typename TTypes::ConstMatrix paddings = in1.matrix();
     for (int d = 0; d < dims; ++d) {
-      const int32 before = paddings(d, 0);  // Pad before existing elements.
-      const int32 after = paddings(d, 1);   // Pad after existing elements.
+      const Tpaddings before = paddings(d, 0);  // Pad before existing elements.
+      const Tpaddings after = paddings(d, 1);   // Pad after existing elements.
       OP_REQUIRES(context, before >= 0 && after >= 0,
                   errors::InvalidArgument("paddings must be non-negative: ",
                                           before, " ", after));
@@ -121,7 +121,7 @@ class MirrorPadOp : public OpKernel {
 
 #define MIRROR_PAD_CASE(i)                                                \
   case i: {                                                               \
-    functor::MirrorPad()(                                   \
+    functor::MirrorPad()(                        \
         context->eigen_device(), To32Bit(output->tensor()), \
         To32Bit(in0.tensor()), paddings, offset_);                  \
     break;                                                                \
@@ -152,20 +152,25 @@ using GpuDevice = Eigen::GpuDevice;
 namespace functor {
 // Forward declarations of the functor specializations defined in the sharded
 // files.
-#define DECLARE_CPU_SPEC(T, i)                                               \
-  template <>                                                                \
-  void MirrorPad::operator()(                               \
-      const CpuDevice&, typename TTypes::Tensor,                \
-      typename TTypes::ConstTensor, TTypes::ConstMatrix, \
-      int);                                                                  \
-  extern template struct MirrorPad;
-
-#define DECLARE_CPU_SPECS(T) \
-  DECLARE_CPU_SPEC(T, 1);    \
-  DECLARE_CPU_SPEC(T, 2);    \
-  DECLARE_CPU_SPEC(T, 3);    \
-  DECLARE_CPU_SPEC(T, 4);    \
-  DECLARE_CPU_SPEC(T, 5);
+#define DECLARE_CPU_SPEC(T, Tpaddings, i)                     \
+  template <>                                                 \
+  void MirrorPad::operator()(     \
+      const CpuDevice&, typename TTypes::Tensor, \
+      typename TTypes::ConstTensor,              \
+      TTypes::ConstMatrix, int);                   \
+  extern template struct MirrorPad;
+
+#define DECLARE_CPU_SPECS(T)     \
+  DECLARE_CPU_SPEC(T, int32, 1); \
+  DECLARE_CPU_SPEC(T, int32, 2); \
+  DECLARE_CPU_SPEC(T, int32, 3); \
+  DECLARE_CPU_SPEC(T, int32, 4); \
+  DECLARE_CPU_SPEC(T, int32, 5); \
+  DECLARE_CPU_SPEC(T, int64, 1); \
+  DECLARE_CPU_SPEC(T, int64, 2); \
+  DECLARE_CPU_SPEC(T, int64, 3); \
+  DECLARE_CPU_SPEC(T, int64, 4); \
+  DECLARE_CPU_SPEC(T, int64, 5);
 
 TF_CALL_POD_TYPES(DECLARE_CPU_SPECS);
 
@@ -179,7 +184,13 @@ TF_CALL_POD_TYPES(DECLARE_CPU_SPECS);
                               .TypeConstraint("T")          \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadOp);
+                          MirrorPadOp);   \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPad")                       \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint("T")          \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadOp);
 
 // Note that we do register for bool type, but not in the gradient op.
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
@@ -188,20 +199,25 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 namespace functor {
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPEC(T, i)                                               \
-  template <>                                                                \
-  void MirrorPad::operator()(                               \
-      const GpuDevice&, typename TTypes::Tensor,                \
-      typename TTypes::ConstTensor, TTypes::ConstMatrix, \
-      int);                                                                  \
-  extern template struct MirrorPad;
-
-#define DECLARE_GPU_SPECS(T) \
-  DECLARE_GPU_SPEC(T, 1);    \
-  DECLARE_GPU_SPEC(T, 2);    \
-  DECLARE_GPU_SPEC(T, 3);    \
-  DECLARE_GPU_SPEC(T, 4);    \
-  DECLARE_GPU_SPEC(T, 5);
+#define DECLARE_GPU_SPEC(T, Tpaddings, i)                     \
+  template <>                                                 \
+  void MirrorPad::operator()(     \
+      const GpuDevice&, typename TTypes::Tensor, \
+      typename TTypes::ConstTensor,              \
+      TTypes::ConstMatrix, int);                   \
+  extern template struct MirrorPad;
+
+#define DECLARE_GPU_SPECS(T)     \
+  DECLARE_GPU_SPEC(T, int32, 1); \
+  DECLARE_GPU_SPEC(T, int32, 2); \
+  DECLARE_GPU_SPEC(T, int32, 3); \
+  DECLARE_GPU_SPEC(T, int32, 4); \
+  DECLARE_GPU_SPEC(T, int32, 5); \
+  DECLARE_GPU_SPEC(T, int64, 1); \
+  DECLARE_GPU_SPEC(T, int64, 2); \
+  DECLARE_GPU_SPEC(T, int64, 3); \
+  DECLARE_GPU_SPEC(T, int64, 4); \
+  DECLARE_GPU_SPEC(T, int64, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPECS
@@ -215,14 +231,20 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint("T")             \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadOp)
+                          MirrorPadOp);      \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPad")                       \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint("T")             \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadOp);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 #endif  // GOOGLE_CUDA
 
 // Gradient op.
-template 
+template 
 class MirrorPadGradOp : public OpKernel {
  public:
   explicit MirrorPadGradOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -269,10 +291,10 @@ class MirrorPadGradOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes::ConstMatrix paddings = in1.matrix();
+    typename TTypes::ConstMatrix paddings = in1.matrix();
     for (int d = 0; d < dims; ++d) {
-      const int32 before = paddings(d, 0);  // Pad before existing elements.
-      const int32 after = paddings(d, 1);   // Pad after existing elements.
+      const Tpaddings before = paddings(d, 0);  // Pad before existing elements.
+      const Tpaddings after = paddings(d, 1);   // Pad after existing elements.
       OP_REQUIRES(context, before >= 0 && after >= 0,
                   errors::InvalidArgument("Paddings must be non-negative: ",
                                           before, ", ", after));
@@ -308,7 +330,7 @@ class MirrorPadGradOp : public OpKernel {
 
 #define MIRROR_PAD_GRAD_CASE(k)                                           \
   case k: {                                                               \
-    functor::MirrorPadGrad()(                               \
+    functor::MirrorPadGrad()(                    \
         context->eigen_device(), To32Bit(output->tensor()), \
         To32Bit(in0.tensor()), paddings, offset_,                   \
         To32Bit(scratch.tensor()));                                 \
@@ -337,33 +359,45 @@ class MirrorPadGradOp : public OpKernel {
 namespace functor {
 // Forward declarations of the functor specializations defined in the sharded
 // files.
-#define DECLARE_CPU_SPEC(T, k)                                               \
-  template <>                                                                \
-  void MirrorPadGrad::operator()(                           \
-      const CpuDevice&, typename TTypes::Tensor,                \
-      typename TTypes::ConstTensor, TTypes::ConstMatrix, \
-      int, typename TTypes::Tensor);                            \
-  extern template struct MirrorPadGrad;
-
-#define DECLARE_CPU_SPECS(T) \
-  DECLARE_CPU_SPEC(T, 1);    \
-  DECLARE_CPU_SPEC(T, 2);    \
-  DECLARE_CPU_SPEC(T, 3);    \
-  DECLARE_CPU_SPEC(T, 4);    \
-  DECLARE_CPU_SPEC(T, 5);
+#define DECLARE_CPU_SPEC(T, Tpaddings, k)                     \
+  template <>                                                 \
+  void MirrorPadGrad::operator()( \
+      const CpuDevice&, typename TTypes::Tensor, \
+      typename TTypes::ConstTensor,              \
+      TTypes::ConstMatrix, int,                    \
+      typename TTypes::Tensor);                  \
+  extern template struct MirrorPadGrad;
+
+#define DECLARE_CPU_SPECS(T)     \
+  DECLARE_CPU_SPEC(T, int32, 1); \
+  DECLARE_CPU_SPEC(T, int32, 2); \
+  DECLARE_CPU_SPEC(T, int32, 3); \
+  DECLARE_CPU_SPEC(T, int32, 4); \
+  DECLARE_CPU_SPEC(T, int32, 5); \
+  DECLARE_CPU_SPEC(T, int64, 1); \
+  DECLARE_CPU_SPEC(T, int64, 2); \
+  DECLARE_CPU_SPEC(T, int64, 3); \
+  DECLARE_CPU_SPEC(T, int64, 4); \
+  DECLARE_CPU_SPEC(T, int64, 5);
 
 TF_CALL_NUMBER_TYPES(DECLARE_CPU_SPECS);
 #undef DECLARE_CPU_SPECS
 #undef DECLARE_CPU_SPEC
 }  // namespace functor
 
-#define REGISTER_KERNEL(type)                                     \
-  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                   \
-                              .Device(DEVICE_CPU)                 \
-                              .TypeConstraint("T")          \
-                              .TypeConstraint("Tpaddings") \
-                              .HostMemory("paddings"),            \
-                          MirrorPadGradOp);
+#define REGISTER_KERNEL(type)                                       \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                     \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint("T")            \
+                              .TypeConstraint("Tpaddings")   \
+                              .HostMemory("paddings"),              \
+                          MirrorPadGradOp); \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                     \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint("T")            \
+                              .TypeConstraint("Tpaddings")   \
+                              .HostMemory("paddings"),              \
+                          MirrorPadGradOp);
 
 TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
@@ -371,20 +405,26 @@ TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 namespace functor {
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPEC(T, k)                                               \
-  template <>                                                                \
-  void MirrorPadGrad::operator()(                           \
-      const GpuDevice&, typename TTypes::Tensor,                \
-      typename TTypes::ConstTensor, TTypes::ConstMatrix, \
-      int, typename TTypes::Tensor);                            \
-  extern template struct MirrorPadGrad;
-
-#define DECLARE_GPU_SPECS(T) \
-  DECLARE_GPU_SPEC(T, 1);    \
-  DECLARE_GPU_SPEC(T, 2);    \
-  DECLARE_GPU_SPEC(T, 3);    \
-  DECLARE_GPU_SPEC(T, 4);    \
-  DECLARE_GPU_SPEC(T, 5);
+#define DECLARE_GPU_SPEC(T, Tpaddings, k)                     \
+  template <>                                                 \
+  void MirrorPadGrad::operator()( \
+      const GpuDevice&, typename TTypes::Tensor, \
+      typename TTypes::ConstTensor,              \
+      TTypes::ConstMatrix, int,                    \
+      typename TTypes::Tensor);                  \
+  extern template struct MirrorPadGrad;
+
+#define DECLARE_GPU_SPECS(T)     \
+  DECLARE_GPU_SPEC(T, int32, 1); \
+  DECLARE_GPU_SPEC(T, int32, 2); \
+  DECLARE_GPU_SPEC(T, int32, 3); \
+  DECLARE_GPU_SPEC(T, int32, 4); \
+  DECLARE_GPU_SPEC(T, int32, 5); \
+  DECLARE_GPU_SPEC(T, int64, 1); \
+  DECLARE_GPU_SPEC(T, int64, 2); \
+  DECLARE_GPU_SPEC(T, int64, 3); \
+  DECLARE_GPU_SPEC(T, int64, 4); \
+  DECLARE_GPU_SPEC(T, int64, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPECS
@@ -398,7 +438,13 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint("T")             \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadGradOp)
+                          MirrorPadGradOp);  \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                   \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint("T")             \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadGradOp);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
diff --git a/tensorflow/core/kernels/mirror_pad_op.h b/tensorflow/core/kernels/mirror_pad_op.h
index b83d2223d0..81150a9e79 100644
--- a/tensorflow/core/kernels/mirror_pad_op.h
+++ b/tensorflow/core/kernels/mirror_pad_op.h
@@ -64,9 +64,8 @@ class TensorMirrorPadOp
       StorageKind;
   typedef typename Eigen::internal::traits::Index Index;
 
-  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
-  TensorMirrorPadOp(const XprType& expr, const PaddingDimensions& padding_dims,
-                    Index offset)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorMirrorPadOp(
+      const XprType& expr, const PaddingDimensions& padding_dims, Index offset)
       : xpr_(expr), padding_dims_(padding_dims), offset_(offset) {}
 
   EIGEN_DEVICE_FUNC
@@ -336,12 +335,12 @@ namespace functor {
 
 // offset argument must be either 0 or 1. This controls whether the boundary
 // values are replicated (offset == 0) or not replicated (offset == 1).
-template 
+template 
 struct MirrorPad {
   void operator()(const Device& device,
                   typename TTypes::Tensor output,
                   typename TTypes::ConstTensor input,
-                  TTypes::ConstMatrix padding, int offset) {
+                  typename TTypes::ConstMatrix padding, int offset) {
     Eigen::array, Dims> padding_dims;
 
     for (int i = 0; i < Dims; ++i) {
@@ -363,12 +362,12 @@ struct MirrorPad {
 
 // offset argument must be either 0 or 1. This controls whether the boundary
 // values are replicated (offset == 0) or not replicated (offset == 1).
-template 
+template 
 struct MirrorPadGrad {
   void operator()(const Device& device,
                   typename TTypes::Tensor output,
                   typename TTypes::ConstTensor input,
-                  TTypes::ConstMatrix paddings, int offset,
+                  typename TTypes::ConstMatrix paddings, int offset,
                   typename TTypes::Tensor scratch) {
     // Copy the gradient input into the scratch buffer.
     scratch.device(device) = input;
diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
index 9864f5633a..bb22b2aa91 100644
--- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
+++ b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
@@ -25,13 +25,17 @@ namespace tensorflow {
 
 using CpuDevice = Eigen::ThreadPoolDevice;
 
-#define DEFINE_CPU_SPECS(T) \
-  template struct functor::MirrorPad;
+#define DEFINE_CPU_SPECS(T)                                                    \
+  template struct functor::MirrorPad; \
+  template struct functor::MirrorPad;
 TF_CALL_POD_TYPES(DEFINE_CPU_SPECS);
 #undef DEFINE_CPU_SPECS
 
-#define DEFINE_CPU_SPECS(T) \
-  template struct functor::MirrorPadGrad;
+#define DEFINE_CPU_SPECS(T)                                   \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad;
 TF_CALL_NUMBER_TYPES(DEFINE_CPU_SPECS);
 #undef DEFINE_CPU_SPECS
 
diff --git a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
index 8074aa9624..dbd0a9bd8f 100644
--- a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
@@ -25,17 +25,27 @@ namespace tensorflow {
 
 using GpuDevice = Eigen::GpuDevice;
 
-#define DEFINE_GPU_SPECS(T)                                \
-  template struct functor::MirrorPad;     \
-  template struct functor::MirrorPad;     \
-  template struct functor::MirrorPad;     \
-  template struct functor::MirrorPad;     \
-  template struct functor::MirrorPad;     \
-  template struct functor::MirrorPadGrad; \
-  template struct functor::MirrorPadGrad; \
-  template struct functor::MirrorPadGrad; \
-  template struct functor::MirrorPadGrad; \
-  template struct functor::MirrorPadGrad;
+#define DEFINE_GPU_SPECS(T)                                       \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPad;     \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad; \
+  template struct functor::MirrorPadGrad;
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
 #undef DEFINE_GPU_SPECS
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 57661e8b10..369f632fb4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -288,8 +288,10 @@ class MklConv2DOp : public OpKernel {
     mkl_filter_output_mkl_shape.SetMklLayout(mkl_context.prim_fwd,
                                              dnnResourceFilter);
 
-    size_t filter_sizes[4] = {filter.dim_size(0), filter.dim_size(1),
-                              filter.dim_size(2), filter.dim_size(3)};
+    size_t filter_sizes[4] = {static_cast(filter.dim_size(0)),
+                              static_cast(filter.dim_size(1)),
+                              static_cast(filter.dim_size(2)),
+                              static_cast(filter.dim_size(3))};
     mkl_filter_output_mkl_shape.SetTfLayout(filter.dims(), filter_sizes,
                                             mkl_context.filter_strides);
 
diff --git a/tensorflow/core/kernels/nth_element_op.cc b/tensorflow/core/kernels/nth_element_op.cc
new file mode 100644
index 0000000000..da825e408c
--- /dev/null
+++ b/tensorflow/core/kernels/nth_element_op.cc
@@ -0,0 +1,139 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/nn_ops.cc.
+#include "tensorflow/core/kernels/nth_element_op.h"
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/work_sharder.h"
+#include 
+#include 
+#include 
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template 
+class NthElementOp : public OpKernel {
+ public:
+  explicit NthElementOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("reverse", &reverse_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    // The second args is N, which must be a positive scalar.
+    const auto& n_in = context->input(1);
+    OP_REQUIRES(context, TensorShapeUtils::IsScalar(n_in.shape()),
+                errors::InvalidArgument("N must be scalar, got shape ",
+                                        n_in.shape().DebugString()));
+    int n = n_in.scalar()();
+    OP_REQUIRES(context, n >= 0,
+                errors::InvalidArgument("Need n >= 0, got ", n));
+
+    // The first args is input tensor, which must have 1 dimension at least.
+    const Tensor& input_in = context->input(0);
+    const int num_dims = input_in.dims();
+    OP_REQUIRES(context, num_dims >= 1,
+                errors::InvalidArgument("Input must be >= 1-D, got shape ",
+                                        input_in.shape().DebugString()));
+    // The last dimension of input tensor must be greater than N.
+    OP_REQUIRES(context, input_in.dim_size(num_dims-1) > n,
+                errors::InvalidArgument("Input must have at least n+1 columns"));
+
+    // std::nth_element only support the nth-smallest selection.
+    if (reverse_) {
+      n = input_in.dim_size(num_dims - 1) - n - 1;
+    }
+
+    // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1].
+    TensorShape out_shape;
+    for (int i = 0; i < num_dims-1; ++i) {
+      out_shape.AddDim(input_in.dim_size(i));
+    }
+    Tensor* output_tensor = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, out_shape, &output_tensor));
+
+    functor::NthElementFunctor nthElementFunc;
+    nthElementFunc(context, input_in, *output_tensor, n, reverse_);
+  }
+
+ private:
+  bool reverse_;
+};
+
+namespace functor {
+
+template 
+struct NthElementFunctor {
+  void operator() (OpKernelContext* context,
+                   const Tensor& input_tensor,
+                   Tensor& output_tensor,
+                   int n,
+                   bool reverse) {
+    const T* input = input_tensor.flat().data();
+    T* output = output_tensor.flat().data();
+
+    // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1],
+    // then num_rows = d1*d2...dk-1, last_dim = dk.
+    const int num_rows = output_tensor.NumElements();
+    const int last_dim = input_tensor.dim_size(input_tensor.dims()-1);
+
+    // Allocate each row to different shard.
+    auto SubNthElement = [&, input, output, last_dim, n](int start,
+                                                         int limit) {
+      // std::nth_element would rearrange the array, so we need a new buffer.
+      std::vector buf(last_dim);
+
+      for (int b = start; b < limit; ++b) {
+        // Copy from one row of elements to buffer
+        const T* input_start = input + b * last_dim;
+        const T* input_end = input + (b+1) * last_dim;
+        std::copy(input_start, input_end, buf.begin());
+
+        std::nth_element(buf.begin(), buf.begin()+n, buf.end());
+        // The element placed in the nth position is exactly the element that
+        // would occur in this position if the range was fully sorted.
+        output[b] = buf[n];
+      }
+    };
+
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    // The average time complexity of partition-based nth_element (BFPRT) is O(n),
+    // althought the worst time complexity could be O(n^2).
+    // Here, 20 is a empirical factor of cost_per_unit.
+    Shard(worker_threads.num_threads, worker_threads.workers, num_rows,
+          20 * last_dim, SubNthElement);
+  }
+};
+
+}  // namespace functor
+
+
+#define REGISTER_NTHOP(T)                                           \
+  REGISTER_KERNEL_BUILDER(                                          \
+      Name("NthElement").Device(DEVICE_CPU).TypeConstraint("T"), \
+      NthElementOp)
+
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_NTHOP);
+#undef REGISTER_NTHOP
+
+}  // end namespace tensorflow
+
diff --git a/tensorflow/core/kernels/nth_element_op.h b/tensorflow/core/kernels/nth_element_op.h
new file mode 100644
index 0000000000..11a6c996b0
--- /dev/null
+++ b/tensorflow/core/kernels/nth_element_op.h
@@ -0,0 +1,39 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_NTH_ELEMENT_OP_H_
+#define TENSORFLOW_NTH_ELEMENT_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template 
+struct NthElementFunctor {
+  void operator() (OpKernelContext* context,
+                   const Tensor& input_tensor,
+                   Tensor& output_tensor,
+                   int n);
+};
+
+}  // namespace functor
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_NTH_ELEMENT_OP_H_
diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc
index 6196c5ed93..eff3e4d92c 100644
--- a/tensorflow/core/kernels/pad_op.cc
+++ b/tensorflow/core/kernels/pad_op.cc
@@ -40,9 +40,9 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
-template 
+template 
 class PadOp : public OpKernel {
  public:
   explicit PadOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -82,10 +82,11 @@ class PadOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes::ConstMatrix paddings = in1.matrix();
+    typename TTypes::ConstMatrix paddings = in1.matrix();
     for (int d = 0; d < fixed_dims; ++d) {
-      const int32 before_d = paddings(d, 0);  // Pad before existing elements.
-      const int32 after_d = paddings(d, 1);   // Pad after existing elements.
+      const Tpadding before_d =
+          paddings(d, 0);                       // Pad before existing elements.
+      const Tpadding after_d = paddings(d, 1);  // Pad after existing elements.
       OP_REQUIRES(context, before_d >= 0 && after_d >= 0,
                   errors::InvalidArgument("Paddings must be non-negative: ",
                                           before_d, " ", after_d));
@@ -142,32 +143,47 @@ class PadOp : public OpKernel {
   template 
   void Operate(OpKernelContext* context,
                typename TTypes::ConstTensor input,
-               TTypes::ConstMatrix paddings, T pad_value,
+               typename TTypes::ConstMatrix paddings, T pad_value,
                Tensor* output) {
     CHECK_EQ(Dims, paddings.dimension(0));
     CHECK_EQ(2, paddings.dimension(1));
-    Eigen::array, Dims> paddings_array;
+    Eigen::array, Dims> paddings_array;
     for (int i = 0; i < Dims; ++i) {
       paddings_array[i] = {paddings(i, 0), paddings(i, 1)};
     }
-    functor::Pad functor;
+    functor::Pad functor;
     functor(context->eigen_device(), output->tensor(), input,
             paddings_array, pad_value);
   }
 };
 
-#define REGISTER_KERNEL(type)                                 \
-  REGISTER_KERNEL_BUILDER(Name("Pad")                         \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint("T")      \
-                              .HostMemory("paddings"),        \
-                          PadOp);            \
-  REGISTER_KERNEL_BUILDER(Name("PadV2")                       \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint("T")      \
-                              .HostMemory("paddings")         \
-                              .HostMemory("constant_values"), \
-                          PadOp);
+#define REGISTER_KERNEL(type)                                     \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint("T")          \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp);         \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint("T")          \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp);         \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint("T")          \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp);         \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint("T")          \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp);
 
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
@@ -177,11 +193,17 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL);
 namespace functor {
 #define DECLARE_GPU_SPEC(T, Dims)                                         \
   template <>                                                             \
-  void Pad::operator()(                               \
+  void Pad::operator()(                        \
       const GPUDevice& d, typename TTypes::Tensor output,        \
       typename TTypes::ConstTensor input,                        \
       Eigen::array, Dims> paddings, T pad_value); \
-  extern template struct Pad;
+  extern template struct Pad;                  \
+  template <>                                                             \
+  void Pad::operator()(                        \
+      const GPUDevice& d, typename TTypes::Tensor output,        \
+      typename TTypes::ConstTensor input,                        \
+      Eigen::array, Dims> paddings, T pad_value); \
+  extern template struct Pad;
 
 #define DECLARE_GPU_SPECS(T) \
   DECLARE_GPU_SPEC(T, 0);    \
@@ -202,14 +224,27 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint("T")             \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          PadOp);                   \
+                          PadOp);            \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint("T")             \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp);            \
   REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
                               .Device(DEVICE_GPU)                 \
                               .TypeConstraint("T")             \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings")             \
                               .HostMemory("constant_values"),     \
-                          PadOp)
+                          PadOp)             \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint("T")             \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 
@@ -223,7 +258,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad")
                             .HostMemory("input")
                             .HostMemory("paddings")
                             .HostMemory("output"),
-                        PadOp);
+                        PadOp);
+REGISTER_KERNEL_BUILDER(Name("Pad")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint("T")
+                            .TypeConstraint("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("output"),
+                        PadOp);
 REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .Device(DEVICE_GPU)
                             .TypeConstraint("T")
@@ -232,7 +275,16 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .HostMemory("paddings")
                             .HostMemory("constant_values")
                             .HostMemory("output"),
-                        PadOp);
+                        PadOp);
+REGISTER_KERNEL_BUILDER(Name("PadV2")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint("T")
+                            .TypeConstraint("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("constant_values")
+                            .HostMemory("output"),
+                        PadOp);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
@@ -243,14 +295,27 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                               .TypeConstraint("T")             \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          PadOp);                  \
+                          PadOp);           \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_SYCL)                \
+                              .TypeConstraint("T")             \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp);           \
   REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
                               .Device(DEVICE_SYCL)                \
                               .TypeConstraint("T")             \
                               .TypeConstraint("Tpaddings") \
                               .HostMemory("paddings")             \
                               .HostMemory("constant_values"),     \
-                          PadOp)
+                          PadOp)            \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_SYCL)                \
+                              .TypeConstraint("T")             \
+                              .TypeConstraint("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp)
 
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNEL);
 REGISTER_KERNEL_BUILDER(Name("Pad")
@@ -260,7 +325,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad")
                             .HostMemory("input")
                             .HostMemory("paddings")
                             .HostMemory("output"),
-                        PadOp);
+                        PadOp);
+REGISTER_KERNEL_BUILDER(Name("Pad")
+                            .Device(DEVICE_SYCL)
+                            .TypeConstraint("T")
+                            .TypeConstraint("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("output"),
+                        PadOp);
 REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .Device(DEVICE_SYCL)
                             .TypeConstraint("T")
@@ -269,8 +342,17 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .HostMemory("paddings")
                             .HostMemory("constant_values")
                             .HostMemory("output"),
-                        PadOp);
+                        PadOp);
+REGISTER_KERNEL_BUILDER(Name("PadV2")
+                            .Device(DEVICE_SYCL)
+                            .TypeConstraint("T")
+                            .TypeConstraint("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("constant_values")
+                            .HostMemory("output"),
+                        PadOp);
 #undef REGISTER_SYCL_KERNEL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/pad_op.h b/tensorflow/core/kernels/pad_op.h
index 95a7c9a3ae..ee9e0f0330 100644
--- a/tensorflow/core/kernels/pad_op.h
+++ b/tensorflow/core/kernels/pad_op.h
@@ -25,13 +25,13 @@ namespace tensorflow {
 namespace functor {
 
 // Functor used by PadOp to do the computations.
-template 
+template 
 struct Pad {
   // Pad "input" into "output", as specified by "paddings" and "pad_value".
   // See pad_op.cc for details.
   void operator()(const Device& d, typename TTypes::Tensor output,
                   typename TTypes::ConstTensor input,
-                  Eigen::array, Dims> paddings,
+                  Eigen::array, Dims> paddings,
                   T pad_value) {
     if (Eigen::internal::is_same::value &&
         (output.size() <= std::numeric_limits::max())) {
@@ -42,12 +42,12 @@ struct Pad {
   }
 };
 
-template 
-struct Pad {
+template 
+struct Pad {
   // In the scalar case we simply copy the input.
   void operator()(const Device& d, typename TTypes::Tensor output,
                   typename TTypes::ConstTensor input,
-                  Eigen::array, 0>, T) {
+                  Eigen::array, 0>, T) {
     output.device(d) = input;
   }
 };
diff --git a/tensorflow/core/kernels/pad_op_gpu.cu.cc b/tensorflow/core/kernels/pad_op_gpu.cu.cc
index f98631df17..613ad62825 100644
--- a/tensorflow/core/kernels/pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/pad_op_gpu.cu.cc
@@ -26,14 +26,18 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Definition of the GPU implementations declared in pad_op.cc.
-#define DEFINE_GPU_SPECS(T)                      \
-  template struct functor::Pad; \
-  template struct functor::Pad; \
-  template struct functor::Pad; \
-  template struct functor::Pad; \
-  template struct functor::Pad; \
-  template struct functor::Pad; \
-  template struct functor::Pad;
+#define DEFINE_GPU_PAD_SPECS(T, Tpadding)                  \
+  template struct functor::Pad; \
+  template struct functor::Pad; \
+  template struct functor::Pad; \
+  template struct functor::Pad; \
+  template struct functor::Pad; \
+  template struct functor::Pad; \
+  template struct functor::Pad;
+
+#define DEFINE_GPU_SPECS(T)      \
+  DEFINE_GPU_PAD_SPECS(T, int32) \
+  DEFINE_GPU_PAD_SPECS(T, int64)
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
 
diff --git a/tensorflow/core/kernels/reduction_ops_all.cc b/tensorflow/core/kernels/reduction_ops_all.cc
index 41abc2b957..4a34c4ef51 100644
--- a/tensorflow/core/kernels/reduction_ops_all.cc
+++ b/tensorflow/core/kernels/reduction_ops_all.cc
@@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint("Tidx")
         .Device(DEVICE_CPU)
         .HostMemory("reduction_indices"),
-    ReductionOp);
+    ReductionOp);
+REGISTER_KERNEL_BUILDER(
+    Name("All")
+        .TypeConstraint("Tidx")
+        .Device(DEVICE_CPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp);
 
 #if GOOGLE_CUDA
 REGISTER_KERNEL_BUILDER(
@@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint("Tidx")
         .Device(DEVICE_GPU)
         .HostMemory("reduction_indices"),
-    ReductionOp);
+    ReductionOp);
+REGISTER_KERNEL_BUILDER(
+    Name("All")
+        .TypeConstraint("Tidx")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp);
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_any.cc b/tensorflow/core/kernels/reduction_ops_any.cc
index a2087cc3b7..6c0519de95 100644
--- a/tensorflow/core/kernels/reduction_ops_any.cc
+++ b/tensorflow/core/kernels/reduction_ops_any.cc
@@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint("Tidx")
         .Device(DEVICE_CPU)
         .HostMemory("reduction_indices"),
-    ReductionOp);
+    ReductionOp);
+REGISTER_KERNEL_BUILDER(
+    Name("Any")
+        .TypeConstraint("Tidx")
+        .Device(DEVICE_CPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp);
 
 #if GOOGLE_CUDA
 REGISTER_KERNEL_BUILDER(
@@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint("Tidx")
         .Device(DEVICE_GPU)
         .HostMemory("reduction_indices"),
-    ReductionOp);
+    ReductionOp);
+REGISTER_KERNEL_BUILDER(
+    Name("Any")
+        .TypeConstraint("Tidx")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp);
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_common.cc b/tensorflow/core/kernels/reduction_ops_common.cc
index 5eba4288ac..8daab0d6be 100644
--- a/tensorflow/core/kernels/reduction_ops_common.cc
+++ b/tensorflow/core/kernels/reduction_ops_common.cc
@@ -57,13 +57,12 @@ gtl::InlinedVector ReductionHelper::permutation() {
   return perm;
 }
 
-Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
-                                 const bool keep_dims) {
-  // bitmap[i] indicates whether to reduce data along i-th axis.
-  gtl::InlinedVector bitmap(data.dims(), false);
-  auto axis_vec = axis.flat();
+template 
+Status SimplifyHelper(const Tensor& data, const Tensor& axis,
+                      gtl::InlinedVector& bitmap) {
+  auto axis_vec = axis.flat();
   for (int64 i = 0; i < axis.NumElements(); ++i) {
-    int32 index = axis_vec(i);
+    Tperm index = axis_vec(i);
     if (index < -data.dims() || index >= data.dims()) {
       return errors::InvalidArgument("Invalid reduction dimension (", index,
                                      " for input with ", data.dims(),
@@ -72,7 +71,18 @@ Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
     index = (index + data.dims()) % data.dims();
     bitmap[index] = true;
   }
+  return Status::OK();
+}
 
+Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
+                                 const bool keep_dims) {
+  // bitmap[i] indicates whether to reduce data along i-th axis.
+  gtl::InlinedVector bitmap(data.dims(), false);
+  if (axis.dtype() == DT_INT32) {
+    TF_RETURN_IF_ERROR(SimplifyHelper(data, axis, bitmap));
+  } else {
+    TF_RETURN_IF_ERROR(SimplifyHelper(data, axis, bitmap));
+  }
   // Output tensor's dim sizes.
   out_shape_.clear();
   for (int i = 0; i < data.dims(); ++i) {
diff --git a/tensorflow/core/kernels/reduction_ops_common.h b/tensorflow/core/kernels/reduction_ops_common.h
index 71af9d88dc..9da992ccd1 100644
--- a/tensorflow/core/kernels/reduction_ops_common.h
+++ b/tensorflow/core/kernels/reduction_ops_common.h
@@ -25,6 +25,7 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -42,7 +43,7 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 template 
 struct Constants {
@@ -68,11 +69,13 @@ struct ConstantsBase {
   const Eigen::IndexList> kOne;
   const Eigen::IndexList, Eigen::type2index<2>> kZeroTwo;
 };
-template<> struct Constants : ConstantsBase{};
+template <>
+struct Constants : ConstantsBase {};
 #ifdef TENSORFLOW_USE_SYCL
-template<> struct Constants : ConstantsBase{};
-#endif // TENSORFLOW_USE_SYCL
-#endif // EIGEN_HAS_INDEX_LIST
+template <>
+struct Constants : ConstantsBase {};
+#endif  // TENSORFLOW_USE_SYCL
+#endif  // EIGEN_HAS_INDEX_LIST
 
 class ReductionHelper {
  public:
@@ -131,12 +134,13 @@ class ReductionHelper {
 
 // For operations where the output is a reduction function along some
 // dimensions of the input.
-template 
+template 
 class ReductionOp : public OpKernel {
  public:
   explicit ReductionOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
     const DataType dt = DataTypeToEnum::v();
-    OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, DT_INT32}, {dt}));
+    const DataType pt = DataTypeToEnum::v();
+    OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, pt}, {dt}));
 
     OP_REQUIRES_OK(ctx, ctx->GetAttr("keep_dims", &keep_dims_));
   }
@@ -266,20 +270,19 @@ struct ReduceFunctorBase {
   }
 
   template 
-  static void FillIdentity(const Device& d, OUT_T out,
-                           const Reducer& reducer) {
+  static void FillIdentity(const Device& d, OUT_T out, const Reducer& reducer) {
     FillIdentityEigenImpl(d, out, reducer);
   }
 };
 
 template 
 struct ReduceFunctor
-        : ReduceFunctorBase{};
+    : ReduceFunctorBase {};
 #if TENSORFLOW_USE_SYCL
 template 
 struct ReduceFunctor
-        : ReduceFunctorBase{};
-#endif // TENSORFLOW_USE_SYCL
+    : ReduceFunctorBase {};
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace functor
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_max.cc b/tensorflow/core/kernels/reduction_ops_max.cc
index 4ca5c11a48..9cf953f4bf 100644
--- a/tensorflow/core/kernels/reduction_ops_max.cc
+++ b/tensorflow/core/kernels/reduction_ops_max.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Max")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ReductionOp>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx"),                                      \
+      ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx"),                                      \
+      ReductionOp>);
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Max")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp>);
 REGISTER_GPU_KERNELS(float);
 REGISTER_GPU_KERNELS(double);
 REGISTER_GPU_KERNELS(int64);
@@ -52,21 +65,37 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint("T")
         .TypeConstraint("Tidx"),
-    ReductionOp>);
+    ReductionOp>);
+REGISTER_KERNEL_BUILDER(
+    Name("Max")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint("T")
+        .TypeConstraint("Tidx"),
+    ReductionOp>);
 
 #undef REGISTER_GPU_KERNELS
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Max")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Max")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint("T")                   \
+                              .TypeConstraint("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Max")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint("T")                   \
+                              .TypeConstraint("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -78,8 +107,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint("T")
         .TypeConstraint("Tidx"),
-    ReductionOp>);
+    ReductionOp>);
+REGISTER_KERNEL_BUILDER(
+    Name("Max")
+        .Device(DEVICE_SYCL)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint("T")
+        .TypeConstraint("Tidx"),
+    ReductionOp>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_mean.cc b/tensorflow/core/kernels/reduction_ops_mean.cc
index 5b01de8ddb..f61589f913 100644
--- a/tensorflow/core/kernels/reduction_ops_mean.cc
+++ b/tensorflow/core/kernels/reduction_ops_mean.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Mean")                        \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ReductionOp>);
+#define REGISTER_CPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx"),               \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx"),               \
+                          ReductionOp>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Mean")                          \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_GPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
 TF_CALL_complex128(REGISTER_GPU_KERNELS);
@@ -45,17 +58,24 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Mean")                          \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_SYCL_KERNELS(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc
index 1e394bea41..807ac0a456 100644
--- a/tensorflow/core/kernels/reduction_ops_min.cc
+++ b/tensorflow/core/kernels/reduction_ops_min.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Min")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ReductionOp>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx"),                                      \
+      ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx"),                                      \
+      ReductionOp>);
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Min")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp>);
 REGISTER_GPU_KERNELS(float);
 REGISTER_GPU_KERNELS(double);
 
@@ -51,21 +64,37 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint("T")
         .TypeConstraint("Tidx"),
-    ReductionOp>);
+    ReductionOp>);
+REGISTER_KERNEL_BUILDER(
+    Name("Min")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint("T")
+        .TypeConstraint("Tidx"),
+    ReductionOp>);
 
 #undef REGISTER_GPU_KERNELS
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Min")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Min")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint("T")                   \
+                              .TypeConstraint("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Min")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint("T")                   \
+                              .TypeConstraint("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint("T")
         .TypeConstraint("Tidx"),
-    ReductionOp>);
+    ReductionOp>);
+REGISTER_KERNEL_BUILDER(
+    Name("Min")
+        .Device(DEVICE_SYCL)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint("T")
+        .TypeConstraint("Tidx"),
+    ReductionOp>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_prod.cc b/tensorflow/core/kernels/reduction_ops_prod.cc
index 33f6ae6bae..e9b23df746 100644
--- a/tensorflow/core/kernels/reduction_ops_prod.cc
+++ b/tensorflow/core/kernels/reduction_ops_prod.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Prod")                        \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ReductionOp>);
+#define REGISTER_CPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx"),               \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx"),               \
+                          ReductionOp>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Prod")                          \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_GPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_int32(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
@@ -46,18 +59,25 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Prod")                          \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_SYCL_KERNELS(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint("T")                    \
+                              .TypeConstraint("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp>);
 REGISTER_SYCL_KERNELS(int32);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_sum.cc b/tensorflow/core/kernels/reduction_ops_sum.cc
index c1f4f3475a..5318d8c133 100644
--- a/tensorflow/core/kernels/reduction_ops_sum.cc
+++ b/tensorflow/core/kernels/reduction_ops_sum.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Sum")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ReductionOp>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx"),                                      \
+      ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx"),                                      \
+      ReductionOp>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Sum")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint("T")                                           \
+          .TypeConstraint("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
 TF_CALL_complex128(REGISTER_GPU_KERNELS);
@@ -53,19 +66,35 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("input")
         .HostMemory("output")
         .HostMemory("reduction_indices"),
-    ReductionOp>);
+    ReductionOp>);
+REGISTER_KERNEL_BUILDER(
+    Name("Sum")
+        .Device(DEVICE_GPU)
+        .TypeConstraint("T")
+        .TypeConstraint("Tidx")
+        .HostMemory("input")
+        .HostMemory("output")
+        .HostMemory("reduction_indices"),
+    ReductionOp>);
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Sum")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint("T")        \
-          .TypeConstraint("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Sum")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint("T")                   \
+                              .TypeConstraint("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp>); \
+  REGISTER_KERNEL_BUILDER(Name("Sum")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint("T")                   \
+                              .TypeConstraint("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("input")
         .HostMemory("output")
         .HostMemory("reduction_indices"),
-    ReductionOp>);
+    ReductionOp>);
+REGISTER_KERNEL_BUILDER(
+    Name("Sum")
+        .Device(DEVICE_SYCL)
+        .TypeConstraint("T")
+        .TypeConstraint("Tidx")
+        .HostMemory("input")
+        .HostMemory("output")
+        .HostMemory("reduction_indices"),
+    ReductionOp>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/resize_bicubic_op.cc b/tensorflow/core/kernels/resize_bicubic_op.cc
index 1c43e77e7c..1a9cf4c640 100644
--- a/tensorflow/core/kernels/resize_bicubic_op.cc
+++ b/tensorflow/core/kernels/resize_bicubic_op.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include 
 #include 
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -29,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/image_resizer_state.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 namespace {
@@ -235,6 +235,7 @@ inline void interpolate_with_caching(
 
   const T* input_b_ptr = input_data.data();
   float* output_y_ptr = output_data.data();
+  std::vector cached_value(num_channels == 3 ? 0 : 4 * num_channels, 0);
 
   for (int64 b = 0; b < resizer_state.batch_size;
        ++b, input_b_ptr += in_batch_width) {
@@ -248,6 +249,7 @@ inline void interpolate_with_caching(
       const T* y_ptr_1 = input_b_ptr + y_wai.index_1 * in_row_width;
       const T* y_ptr_2 = input_b_ptr + y_wai.index_2 * in_row_width;
       const T* y_ptr_3 = input_b_ptr + y_wai.index_3 * in_row_width;
+
       if (num_channels == 3) {
         // Manually unroll case of 3 channels.
         float cached_value_0[4] = {0};
@@ -330,48 +332,61 @@ inline void interpolate_with_caching(
                       x_wai.weight_2, x_wai.weight_3);
         }
       } else {
-        for (int64 c = 0; c < num_channels; ++c) {
-          float cached_value[4] = {0};
-          for (int64 x = 0; x < resizer_state.out_width; ++x) {
-            const WeightsAndIndices& x_wai = x_wais[x];
-            // Shift values in cached_value to fill first 'advance' values.
-            switch (x_wai.advance) {
-              case 3:
-                cached_value[0] = cached_value[1];
-                cached_value[1] = cached_value[2];
-                cached_value[2] = cached_value[3];
-                break;
-              case 2:
-                cached_value[0] = cached_value[2];
-                cached_value[1] = cached_value[3];
-                break;
-              case 1: {
-                cached_value[0] = cached_value[3];
-                break;
+        for (int64 x = 0; x < resizer_state.out_width; ++x) {
+          const WeightsAndIndices& x_wai = x_wais[x];
+          // Shift values in cached_value to fill first 'advance' values.
+          switch (x_wai.advance) {
+            case 3:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 1];
+                cached_value[4 * c + 1] = cached_value[4 * c + 2];
+                cached_value[4 * c + 2] = cached_value[4 * c + 3];
+              }
+              break;
+            case 2:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 2];
+                cached_value[4 * c + 1] = cached_value[4 * c + 3];
+              }
+              break;
+            case 1: {
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 3];
               }
+              break;
             }
+          }
 
-            // Set the remaining '4-advance' values by computing.
-            switch (x_wai.advance) {
-              case 0:
-                cached_value[0] = ComputeYInterpolation(
+          // Set the remaining '4-advance' values by computing.
+          switch (x_wai.advance) {
+            case 0:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = ComputeYInterpolation(
                     0, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 1:
-                cached_value[1] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 1:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 1] = ComputeYInterpolation(
                     1, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 2:
-                cached_value[2] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 2:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 2] = ComputeYInterpolation(
                     2, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 3:
-                cached_value[3] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 3:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 3] = ComputeYInterpolation(
                     3, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                break;
-            }
+              }
+              break;
+          }
+          for (int64 c = 0; c < num_channels; ++c) {
             output_y_ptr[x * num_channels + c] =
-                Compute(cached_value, x_wai.weight_0, x_wai.weight_1,
+                Compute(&cached_value[4 * c], x_wai.weight_0, x_wai.weight_1,
                         x_wai.weight_2, x_wai.weight_3);
           }
         }
diff --git a/tensorflow/core/kernels/resize_bicubic_op_test.cc b/tensorflow/core/kernels/resize_bicubic_op_test.cc
index ae14d2804e..9e10fec423 100644
--- a/tensorflow/core/kernels/resize_bicubic_op_test.cc
+++ b/tensorflow/core/kernels/resize_bicubic_op_test.cc
@@ -251,14 +251,15 @@ TEST_F(ResizeBicubicOpTest, TestAreaRandomDataSeveralInputsSizes4Channels) {
   RunManyRandomTests(4);
 }
 
-static Graph* ResizeBicubic(int batch_size, int size, int channels) {
+static Graph* ResizeBicubic(int batch_size, int size, int channels,
+                            float scale_y = 0.3, float scale_x = 0.7) {
   Graph* g = new Graph(OpRegistry::Global());
   Tensor input(DT_FLOAT, TensorShape({batch_size, size, size, channels}));
   input.flat().setRandom();
   Tensor shape(DT_INT32, TensorShape({2}));
   auto shape_t = shape.flat();
-  shape_t(0) = 0.3 * size;
-  shape_t(1) = 0.7 * size;
+  shape_t(0) = scale_y * size;
+  shape_t(1) = scale_x * size;
   test::graph::Binary(g, "ResizeBicubic", test::graph::Constant(g, input),
                       test::graph::Constant(g, shape));
   return g;
@@ -285,4 +286,17 @@ BM_ResizeBicubicDev(32, 128, 3);
 BM_ResizeBicubicDev(32, 512, 3);
 BM_ResizeBicubicDev(32, 1024, 3);
 
+#define BM_ResizeBicubicExpand(BATCH, SIZE, CHANNELS)                          \
+  static void BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS(int iters) { \
+    testing::ItemsProcessed(static_cast(iters) * BATCH * SIZE * SIZE *  \
+                            CHANNELS * 8 * 8);                                 \
+    test::Benchmark("cpu", ResizeBicubic(BATCH, SIZE, CHANNELS, 8, 8))         \
+        .Run(iters);                                                           \
+  }                                                                            \
+  BENCHMARK(BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS);
+
+BM_ResizeBicubicExpand(12, 48, 1);
+BM_ResizeBicubicExpand(12, 48, 3);
+BM_ResizeBicubicExpand(12, 48, 40);
+
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/reverse_sequence_op.cc b/tensorflow/core/kernels/reverse_sequence_op.cc
index 505c512cc4..d1980d4b65 100644
--- a/tensorflow/core/kernels/reverse_sequence_op.cc
+++ b/tensorflow/core/kernels/reverse_sequence_op.cc
@@ -175,6 +175,7 @@ class ReverseSequenceOp : public OpKernel {
   REGISTER_REVERSE_SEQUENCE(type, int64);
 
 TF_CALL_NUMBER_TYPES(REGISTER_REVERSE_SEQUENCE_LEN);
+TF_CALL_bool(REGISTER_REVERSE_SEQUENCE_LEN);
 
 #if GOOGLE_CUDA
 
@@ -200,6 +201,7 @@ namespace functor {
   DECLARE_GPU_SPEC_LEN(T, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
+TF_CALL_bool(DECLARE_GPU_SPECS);
 
 }  // namespace functor
 
@@ -215,6 +217,7 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
   REGISTER_REVERSE_SEQUENCE_GPU(type, int64);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_REVERSE_SEQUENCE_GPU_LEN);
+TF_CALL_bool(REGISTER_REVERSE_SEQUENCE_GPU_LEN);
 
 #undef REGISTER_REVERSE_SEQUENCE_GPU
 
diff --git a/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc b/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
index 373fd60687..cb49f14525 100644
--- a/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
@@ -39,6 +39,7 @@ typedef Eigen::GpuDevice GPUDevice;
   DEFINE_GPU_SPEC_LEN(T, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
+TF_CALL_bool(DEFINE_GPU_SPECS);
 
 }  // end namespace tensorflow
 
diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc
index cc434ab0ae..0a6848361a 100644
--- a/tensorflow/core/kernels/scan_ops.cc
+++ b/tensorflow/core/kernels/scan_ops.cc
@@ -35,7 +35,7 @@ namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 
-template 
+template 
 class ScanOp : public OpKernel {
  public:
   explicit ScanOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -51,8 +51,9 @@ class ScanOp : public OpKernel {
                 errors::InvalidArgument("ScanOp: axis must be a scalar, not ",
                                         tensor_axis.shape().DebugString()));
 
-    const int axis_arg = internal::SubtleMustCopy(tensor_axis.scalar()());
-    const int axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg;
+    const Tidx axis_arg =
+        internal::SubtleMustCopy(tensor_axis.scalar()());
+    const Tidx axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg;
     OP_REQUIRES(ctx, FastBoundsCheck(axis, input.dims()),
                 errors::InvalidArgument(
                     "ScanOp: Expected scan axis in the range [", -input.dims(),
@@ -70,11 +71,11 @@ class ScanOp : public OpKernel {
 
     // Dim reduction.
     int64 reduced_shape[3] = {1, 1, 1};
-    for (int i = 0; i < axis; ++i) {
+    for (Tidx i = 0; i < axis; ++i) {
       reduced_shape[0] *= input.dim_size(i);
     }
     reduced_shape[1] = input.dim_size(axis);
-    for (int i = axis + 1; i < input.dims(); ++i) {
+    for (Tidx i = axis + 1; i < input.dims(); ++i) {
       reduced_shape[2] *= input.dim_size(i);
     }
 
@@ -112,51 +113,76 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_FOR_ALL_REDUCERS);
 }  // namespace functor
 #endif  // GOOGLE_CUDA
 
-
 // Register Cumsum kernels
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Cumsum")                      \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ScanOp>)
+#define REGISTER_CPU_KERNELS(type)                                       \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint("T")                                     \
+          .TypeConstraint("Tidx"),                                \
+      ScanOp, int32>) \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint("T")                                     \
+          .TypeConstraint("Tidx"),                                \
+      ScanOp, int64>)
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
-#define REGISTER_GPU_KERNELS(type)       \
-  REGISTER_KERNEL_BUILDER(               \
-      Name("Cumsum")                     \
-          .Device(DEVICE_GPU)            \
-          .TypeConstraint("T")     \
-          .TypeConstraint("Tidx") \
-          .HostMemory("axis"),           \
-      ScanOp>)
+#define REGISTER_GPU_KERNELS(type)                                       \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_GPU)                                            \
+          .TypeConstraint("T")                                     \
+          .TypeConstraint("Tidx")                                 \
+          .HostMemory("axis"),                                           \
+      ScanOp, int32>) \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_GPU)                                            \
+          .TypeConstraint("T")                                     \
+          .TypeConstraint("Tidx")                                 \
+          .HostMemory("axis"),                                           \
+      ScanOp, int64>)
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS)
 #undef REGISTER_GPU_KERNELS
 #endif  // GOOGLE_CUDA
 
 // Register Cumprod kernels
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Cumprod")                     \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint("T")      \
-          .TypeConstraint("Tidx"), \
-      ScanOp>)
+#define REGISTER_CPU_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint("T")                                      \
+          .TypeConstraint("Tidx"),                                 \
+      ScanOp, int32>) \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint("T")                                      \
+          .TypeConstraint("Tidx"),                                 \
+      ScanOp, int64>)
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
-#define REGISTER_GPU_KERNELS(type)       \
-  REGISTER_KERNEL_BUILDER(               \
-      Name("Cumprod")                    \
-          .Device(DEVICE_GPU)            \
-          .TypeConstraint("T")     \
-          .TypeConstraint("Tidx") \
-          .HostMemory("axis"),           \
-      ScanOp>)
+#define REGISTER_GPU_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_GPU)                                             \
+          .TypeConstraint("T")                                      \
+          .TypeConstraint("Tidx")                                  \
+          .HostMemory("axis"),                                            \
+      ScanOp, int32>) \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_GPU)                                             \
+          .TypeConstraint("T")                                      \
+          .TypeConstraint("Tidx")                                  \
+          .HostMemory("axis"),                                            \
+      ScanOp, int64>)
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS)
 #undef REGISTER_GPU_KERNELS
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc
index c8ea923020..e2e3758d87 100644
--- a/tensorflow/core/kernels/sequence_ops.cc
+++ b/tensorflow/core/kernels/sequence_ops.cc
@@ -96,7 +96,7 @@ TF_CALL_double(REGISTER_SYCL_KERNEL);
 TF_CALL_int32(REGISTER_SYCL_KERNEL);
 TF_CALL_int64(REGISTER_SYCL_KERNEL);
 #undef REGISTER_SYCL_KERNEL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
@@ -116,7 +116,7 @@ TF_CALL_int64(REGISTER_GPU_KERNEL);
 #undef REGISTER_CPU_KERNEL
 #undef REGISTER_GPU_KERNEL
 
-template 
+template 
 class LinSpaceOp : public OpKernel {
  public:
   explicit LinSpaceOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -136,7 +136,7 @@ class LinSpaceOp : public OpKernel {
                                         num_in.shape().DebugString()));
     const T start = start_in.scalar()();
     const T stop = stop_in.scalar()();
-    const int32 num = num_in.scalar()();
+    const Tnum num = num_in.scalar()();
     OP_REQUIRES(context, num > 0,
                 errors::InvalidArgument("Requires num > 0: ", num));
     Tensor* out = nullptr;
@@ -147,34 +147,46 @@ class LinSpaceOp : public OpKernel {
       flat(0) = start;
     } else {
       const T step = (stop - start) / (num - 1);
-      for (int32 i = 0; i < num; ++i) flat(i) = start + step * i;
+      for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i;
     }
   }
 };
 
-#define REGISTER_KERNEL(DEV, T)                              \
-  REGISTER_KERNEL_BUILDER(Name("LinSpace")                   \
-                              .Device(DEV)                   \
-                              .TypeConstraint("T")        \
-                              .TypeConstraint("Tidx") \
-                              .HostMemory("start")           \
-                              .HostMemory("stop")            \
-                              .HostMemory("num")             \
-                              .HostMemory("output"),         \
-                          LinSpaceOp);
-#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL(DEVICE_CPU, T)
+#define REGISTER_KERNEL(DEV, T, Tidx)                       \
+  REGISTER_KERNEL_BUILDER(Name("LinSpace")                  \
+                              .Device(DEV)                  \
+                              .TypeConstraint("T")       \
+                              .TypeConstraint("Tidx") \
+                              .HostMemory("start")          \
+                              .HostMemory("stop")           \
+                              .HostMemory("num")            \
+                              .HostMemory("output"),        \
+                          LinSpaceOp);
+
+#define REGISTER_KERNEL_ALL_NUMS(dev, T) \
+  REGISTER_KERNEL(dev, T, int32);        \
+  REGISTER_KERNEL(dev, T, int64)
+
+#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_CPU, T)
 TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
 
 // NOTE(touts): We register the op on GPU but it still runs on CPU
 // because its inputs and outputs are tagged as HostMemory.
-#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL(DEVICE_GPU, T)
+#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_GPU, T)
 TF_CALL_float(REGISTER_GPU_KERNEL);
 TF_CALL_double(REGISTER_GPU_KERNEL);
+#undef REGISTER_GPU_KERNEL
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL(DEVICE_SYCL, T)
+#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_SYCL, T)
 TF_CALL_float(REGISTER_SYCL_KERNEL);
 TF_CALL_double(REGISTER_SYCL_KERNEL);
-#endif // TENSORFLOW_USE_SYCL
+#undef REGISTER_SYCL_KERNEL
+#endif  // TENSORFLOW_USE_SYCL
+
+#undef REGISTER_CPU_KERNEL
+#undef REGISTER_KERNEL_ALL_NUMS
+#undef REGISTER_KERNEL
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc
new file mode 100644
index 0000000000..5f0e0a69a8
--- /dev/null
+++ b/tensorflow/core/kernels/sequence_ops_test.cc
@@ -0,0 +1,148 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+class RangeOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(DataType input_type) {
+    TF_ASSERT_OK(NodeDefBuilder("myop", "Range")
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Finalize(node_def()));
+    TF_ASSERT_OK(InitOp());
+  }
+};
+
+class LinSpaceOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(DataType input_type, DataType index_type) {
+    TF_ASSERT_OK(NodeDefBuilder("myop", "LinSpace")
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(index_type))
+                     .Finalize(node_def()));
+    TF_ASSERT_OK(InitOp());
+  }
+};
+
+TEST_F(RangeOpTest, Simple_D32) {
+  MakeOp(DT_INT32);
+
+  // Feed and run
+  AddInputFromArray(TensorShape({}), {0});
+  AddInputFromArray(TensorShape({}), {10});
+  AddInputFromArray(TensorShape({}), {2});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_INT32, TensorShape({5}));
+  test::FillValues(&expected, {0, 2, 4, 6, 8});
+  test::ExpectTensorEqual(expected, *GetOutput(0));
+}
+
+TEST_F(RangeOpTest, Simple_Float) {
+  MakeOp(DT_FLOAT);
+
+  // Feed and run
+  AddInputFromArray(TensorShape({}), {0.5});
+  AddInputFromArray(TensorShape({}), {2});
+  AddInputFromArray(TensorShape({}), {0.3});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({5}));
+  test::FillValues(&expected, {0.5, 0.8, 1.1, 1.4, 1.7});
+  test::ExpectTensorEqual(expected, *GetOutput(0));
+}
+
+TEST_F(RangeOpTest, Large_Double) {
+  MakeOp(DT_DOUBLE);
+
+  // Feed and run
+  AddInputFromArray(TensorShape({}), {0.0});
+  AddInputFromArray(TensorShape({}), {10000});
+  AddInputFromArray(TensorShape({}), {0.5});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_DOUBLE, TensorShape({20000}));
+  std::vector result;
+  for (int32 i = 0; i < 20000; ++i) result.push_back(i * 0.5);
+  test::FillValues(&expected, gtl::ArraySlice(result));
+  test::ExpectTensorEqual(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Simple_D32) {
+  MakeOp(DT_FLOAT, DT_INT32);
+
+  // Feed and run
+  AddInputFromArray(TensorShape({}), {3.0});
+  AddInputFromArray(TensorShape({}), {7.0});
+  AddInputFromArray(TensorShape({}), {3});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({3}));
+  test::FillValues(&expected, {3.0, 5.0, 7.0});
+  test::ExpectTensorEqual(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Single_D64) {
+  MakeOp(DT_FLOAT, DT_INT64);
+
+  // Feed and run
+  AddInputFromArray(TensorShape({}), {9.0});
+  AddInputFromArray(TensorShape({}), {100.0});
+  AddInputFromArray(TensorShape({}), {1});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({1}));
+  test::FillValues(&expected, {9.0});
+  test::ExpectTensorEqual(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Simple_Double) {
+  MakeOp(DT_DOUBLE, DT_INT32);
+
+  // Feed and run
+  AddInputFromArray(TensorShape({}), {5.0});
+  AddInputFromArray(TensorShape({}), {6.0});
+  AddInputFromArray(TensorShape({}), {6});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_DOUBLE, TensorShape({6}));
+  test::FillValues(&expected, {5.0, 5.2, 5.4, 5.6, 5.8, 6.0});
+  test::ExpectTensorEqual(expected, *GetOutput(0));
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/spacetobatch_op.cc b/tensorflow/core/kernels/spacetobatch_op.cc
index c513683918..95c1f5e7e8 100644
--- a/tensorflow/core/kernels/spacetobatch_op.cc
+++ b/tensorflow/core/kernels/spacetobatch_op.cc
@@ -248,40 +248,34 @@ class SpaceToBatchOp : public OpKernel {
   Tensor block_shape_;
 };
 
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")                     \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tblock_shape") \
-                              .TypeConstraint("Tpaddings")    \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("paddings"),               \
-                          SpaceToBatchNDOp);           \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")                       \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tpaddings")    \
-                              .HostMemory("paddings"),               \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")           \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("paddings"),     \
+                          SpaceToBatchNDOp); \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")             \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("paddings"),     \
                           SpaceToBatchOp);
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER);
 #undef REGISTER
 
 #if GOOGLE_CUDA
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")                     \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tblock_shape") \
-                              .TypeConstraint("Tpaddings")    \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("paddings"),               \
-                          SpaceToBatchNDOp);           \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")                       \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint("T")                \
-                              .TypeConstraint("Tpaddings")    \
-                              .HostMemory("paddings"),               \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")           \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("paddings"),     \
+                          SpaceToBatchNDOp); \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")             \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint("T")      \
+                              .HostMemory("paddings"),     \
                           SpaceToBatchOp);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
diff --git a/tensorflow/core/kernels/sparse_matmul_op.h b/tensorflow/core/kernels/sparse_matmul_op.h
index 308b641b54..cca52558ae 100644
--- a/tensorflow/core/kernels/sparse_matmul_op.h
+++ b/tensorflow/core/kernels/sparse_matmul_op.h
@@ -54,8 +54,9 @@ EIGEN_DEVICE_FUNC inline Packet pexpand_bf16_u(const Packet& from) {
 }
 
 // Specialization non-scalar version on non-sse.
+// Enable vectorization on z13 and higher
 #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) || \
-    defined(EIGEN_VECTORIZE_NEON)
+    defined(EIGEN_VECTORIZE_NEON) || defined(EIGEN_VECTORIZE_ZVECTOR)
 template 
 EIGEN_DEVICE_FUNC inline Packet4f pexpand_bf16_l(const Packet4f& from) {
   float r[4];
@@ -126,8 +127,9 @@ EIGEN_DEVICE_FUNC inline Packet pload2bf16(
 }
 
 // Specialization for pload4bf16 and pload2bf16 for non-sse.
+// Enable vectorization on z13 and higher.
 #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) || \
-    defined(EIGEN_VECTORIZE_NEON)
+    defined(EIGEN_VECTORIZE_NEON) || defined(EIGEN_VECTORIZE_ZVECTOR)
 template <>
 EIGEN_STRONG_INLINE Packet4f pload4bf16(const float* from) {
   tensorflow::uint32 p[4];
diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc
index 1717428adf..0fae46dea6 100644
--- a/tensorflow/core/kernels/stage_op.cc
+++ b/tensorflow/core/kernels/stage_op.cc
@@ -53,7 +53,10 @@ class Buffer : public ResourceBase {
   void notify_inserters_if_bounded(std::unique_lock* lock) {
     if (IsBounded()) {
       lock->unlock();
-      full_cond_var_.notify_one();
+      // Notify all inserters. The removal of an element
+      // may make memory available for many inserters
+      // to insert new elements
+      full_cond_var_.notify_all();
     }
   }
 
@@ -115,9 +118,12 @@ class Buffer : public ResourceBase {
     buf_.push_back(std::move(*tuple));
 
     lock.unlock();
-    // maybe possible to optimize by reducing
-    // how often this signal is sent
-    non_empty_cond_var_.notify_one();
+    // Notify all removers. Removers
+    // may be peeking at a specific element or waiting
+    // for the element at the front of the deque.
+    // As we don't know the appropriate one to wake up
+    // we should wake them all.
+    non_empty_cond_var_.notify_all();
 
     return Status::OK();
   }
diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc
index 79d0c07acd..f6fb0a121d 100644
--- a/tensorflow/core/kernels/stateless_random_ops.cc
+++ b/tensorflow/core/kernels/stateless_random_ops.cc
@@ -137,7 +137,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint("T")                                  \
           .TypeConstraint("dtype"),                              \
       StatelessRandomOp >); \
@@ -146,7 +145,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint("T")                                  \
           .TypeConstraint("dtype"),                              \
       StatelessRandomOp >); \
@@ -155,7 +153,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint("T")                                  \
           .TypeConstraint("dtype"),                              \
       StatelessRandomOp<                                               \
           GPUDevice,                                                   \
diff --git a/tensorflow/core/kernels/tile_functor.h b/tensorflow/core/kernels/tile_functor.h
index 28af2dace3..189be9239b 100644
--- a/tensorflow/core/kernels/tile_functor.h
+++ b/tensorflow/core/kernels/tile_functor.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_KERNELS_TILE_FUNCTOR_H_
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/types.h"
@@ -29,13 +30,13 @@ namespace internal {
 template 
 void TileSimple(const Device& d, Tensor* out, const Tensor& in);
 
-template 
+template 
 void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
-                    const gtl::ArraySlice& broadcast_array) {
+                    const gtl::ArraySlice& broadcast_array) {
   auto x = in.tensor();
   auto y = out->tensor();
 
-  Eigen::array b;
+  Eigen::array b;
   for (int i = 0; i < NDIM; ++i) b[i] = broadcast_array[i];
   if (Eigen::internal::is_same::value) {
     // Use 32bit indexing to speed up the computations
@@ -45,9 +46,9 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
   }
 }
 
-template 
+template 
 void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
-                    const gtl::ArraySlice&) {
+                    const gtl::ArraySlice&) {
   auto x = in.tensor();
   auto y = out->tensor();
   // In the scalar case we simply copy the input.
@@ -58,34 +59,42 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
 
 namespace functor {
 
-template 
+template 
 struct Tile {
   void operator()(const Device& d, Tensor* out, const Tensor& in,
-                  const gtl::ArraySlice broadcast_array) const {
+                  const gtl::ArraySlice broadcast_array) const {
     switch (in.dims()) {
       case 0:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                        broadcast_array);
         break;
       case 1:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       case 2:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       case 3:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       case 4:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       case 5:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       case 6:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       case 7:
-        internal::TileUsingEigen(d, out, in, broadcast_array);
+        internal::TileUsingEigen(d, out, in,
+                                                           broadcast_array);
         break;
       default:
         internal::TileSimple(d, out, in);
diff --git a/tensorflow/core/kernels/tile_functor_cpu.cc b/tensorflow/core/kernels/tile_functor_cpu.cc
index 5952d49221..b2fd669541 100644
--- a/tensorflow/core/kernels/tile_functor_cpu.cc
+++ b/tensorflow/core/kernels/tile_functor_cpu.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
+#include "tensorflow/core/kernels/tile_functor.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/ops_util.h"
-#include "tensorflow/core/kernels/tile_functor.h"
 
 namespace tensorflow {
 
@@ -51,7 +51,9 @@ namespace functor {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
 // Register functors used for Tile functor.
-#define DEFINE_TYPE(T) template struct Tile;
+#define DEFINE_TYPE(T)                       \
+  template struct Tile; \
+  template struct Tile;
 
 TF_CALL_bool(DEFINE_TYPE);
 TF_CALL_float(DEFINE_TYPE);
@@ -70,7 +72,9 @@ TF_CALL_string(DEFINE_TYPE);
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
 
-#define DEFINE_TYPE(T) template struct Tile;
+#define DEFINE_TYPE(T)                        \
+  template struct Tile; \
+  template struct Tile;
 
 TF_CALL_bool(DEFINE_TYPE);
 TF_CALL_float(DEFINE_TYPE);
@@ -81,7 +85,7 @@ TF_CALL_int16(DEFINE_TYPE);
 TF_CALL_int64(DEFINE_TYPE);
 
 #undef DEFINE_TYPE
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // end namespace functor
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.cu.cc
index 1c61c3030a..5a36e7567b 100644
--- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/tile_functor_gpu.cu.cc
@@ -18,10 +18,11 @@ limitations under the License.
 #define EIGEN_USE_GPU
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/tile_functor.h"
+
+#include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/tile_functor.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
-#include "tensorflow/core/framework/register_types.h"
 
 namespace tensorflow {
 namespace internal {
@@ -60,7 +61,8 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) {
     host_buf[ndims + i] = out_strides[i];
     host_buf[ndims * 2 + i] = in.dim_size(i);
   }
-  // Copies the input strides, output strides and input dimension sizes to the device.
+  // Copies the input strides, output strides and input dimension sizes to the
+  // device.
   auto num_bytes = sizeof(int64) * host_buf.size();
   auto dev_buf = d.allocate(num_bytes);
   // NOTE: host_buf is not allocated by CudaHostAllocator, and
@@ -84,7 +86,9 @@ namespace functor {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Register functors used for Tile functor.
-#define DEFINE_TYPE(T) template struct Tile;
+#define DEFINE_TYPE(T)                       \
+  template struct Tile; \
+  template struct Tile;
 
 TF_CALL_int16(DEFINE_TYPE);
 TF_CALL_int32(DEFINE_TYPE);
diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc
index c49ebc0685..4c496a12c2 100644
--- a/tensorflow/core/kernels/tile_ops.cc
+++ b/tensorflow/core/kernels/tile_ops.cc
@@ -42,14 +42,14 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 // Forward declarations of functors that will be defined in tile_ops_impl.h
 namespace functor {
-template 
+template 
 struct Tile {
   void operator()(const Device& d, Tensor* out, const Tensor& in,
-                  const gtl::ArraySlice broadcast_array) const;
+                  const gtl::ArraySlice broadcast_array) const;
 };
 
 template 
@@ -80,7 +80,7 @@ struct ReduceAndReshape {
 }  // namespace functor
 
 // --------------------------------------------------------------------------
-template 
+template 
 class TileOp : public OpKernel {
  public:
   explicit TileOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -105,8 +105,8 @@ class TileOp : public OpKernel {
       return;
     }
 
-    const gtl::ArraySlice multiples_array(multiples.flat().data(),
-                                                 input_dims);
+    const gtl::ArraySlice multiples_array(
+        multiples.flat().data(), input_dims);
     TensorShape output_shape;
     for (int i = 0; i < input_dims; ++i) {
       OP_REQUIRES(
@@ -125,10 +125,10 @@ class TileOp : public OpKernel {
     // If there's no output, there's nothing to do.
     if (output_shape.num_elements() == 0) return;
 
-#define HANDLE_TYPE(DT)                                        \
-  if (context->input(0).dtype() == DT) {                       \
-    HandleCase
(context, multiples_array, result);          \
-    return;                                                    \
+#define HANDLE_TYPE(DT)                               \
+  if (context->input(0).dtype() == DT) {              \
+    HandleCase
(context, multiples_array, result); \
+    return;                                           \
   }
 
 #define HANDLE_TYPE_NAME(T) HANDLE_TYPE(DataTypeToEnum::value)
@@ -158,27 +158,27 @@ class TileOp : public OpKernel {
  private:
   template 
   void HandleCaseImpl(OpKernelContext* context,
-                      const gtl::ArraySlice& multiples_array,
+                      const gtl::ArraySlice& multiples_array,
                       Tensor* result) {
     typedef typename EnumToDataType
::Type T;
-    functor::Tile() (
-        context->eigen_device(), result,
-        context->input(0), multiples_array);
+    functor::Tile()(context->eigen_device(),
+                                           result, context->input(0),
+                                           multiples_array);
   }
 
   template 
   void HandleCase(OpKernelContext* context,
-                  const gtl::ArraySlice& multiples_array,
+                  const gtl::ArraySlice& multiples_array,
                   Tensor* result);
 
   TF_DISALLOW_COPY_AND_ASSIGN(TileOp);
 };
 
-template 
+template 
 template 
-inline void TileOp::HandleCase(
-    OpKernelContext* context, const gtl::ArraySlice& multiples_array,
-    Tensor* result) {
+inline void TileOp::HandleCase(
+    OpKernelContext* context,
+    const gtl::ArraySlice& multiples_array, Tensor* result) {
   // TODO(vrv): print out the device name if useful. Currently disabled to avoid
   // having to use RTTI.
   LOG(FATAL) << "TileOp: Invalid combination of Device, DT: "
@@ -186,25 +186,28 @@ inline void TileOp::HandleCase(
              << DataTypeString(DT);
 }
 
-#define HANDLE_CASE(device, dtype)                                     \
-  template <>                                                          \
-  template <>                                                          \
-  void TileOp::HandleCase(                              \
-      OpKernelContext * context,                                       \
-      const gtl::ArraySlice& multiples_array, Tensor* result) { \
-    HandleCaseImpl(context, multiples_array, result);           \
+#define HANDLE_CASE(device, dtype, Tmultiples)                              \
+  template <>                                                               \
+  template <>                                                               \
+  void TileOp::HandleCase(                       \
+      OpKernelContext * context,                                            \
+      const gtl::ArraySlice& multiples_array, Tensor* result) { \
+    HandleCaseImpl(context, multiples_array, result);                \
   }
 
-#define HANDLE_TYPE_NAME_CPU(T) \
-  HANDLE_CASE(CPUDevice, DataTypeToEnum::value);
+#define HANDLE_TYPE_NAME_CPU(T)                            \
+  HANDLE_CASE(CPUDevice, DataTypeToEnum::value, int32); \
+  HANDLE_CASE(CPUDevice, DataTypeToEnum::value, int64);
 
-#define HANDLE_TYPE_NAME_GPU(T) \
-  HANDLE_CASE(GPUDevice, DataTypeToEnum::value);
+#define HANDLE_TYPE_NAME_GPU(T)                            \
+  HANDLE_CASE(GPUDevice, DataTypeToEnum::value, int32); \
+  HANDLE_CASE(GPUDevice, DataTypeToEnum::value, int64);
 
 #ifdef TENSORFLOW_USE_SYCL
-#define HANDLE_TYPE_NAME_SYCL(T) \
-  HANDLE_CASE(SYCLDevice, DataTypeToEnum::value);
-#endif // TENSORFLOW_USE_SYCL
+#define HANDLE_TYPE_NAME_SYCL(T)                            \
+  HANDLE_CASE(SYCLDevice, DataTypeToEnum::value, int32); \
+  HANDLE_CASE(SYCLDevice, DataTypeToEnum::value, int64);
+#endif  // TENSORFLOW_USE_SYCL
 
 TF_CALL_bool(HANDLE_TYPE_NAME_CPU);
 TF_CALL_float(HANDLE_TYPE_NAME_CPU);
@@ -235,13 +238,13 @@ TF_CALL_double(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int16(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int32(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 #undef HANDLE_TYPE_NAME_CPU
 #undef HANDLE_TYPE_NAME_GPU
 #ifdef TENSORFLOW_USE_SYCL
 #undef HANDLE_TYPE_NAME_SYCL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 #undef HANDLE_CASE
 
 // --------------------------------------------------------------------------
@@ -494,7 +497,7 @@ TF_CALL_int16(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int32(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
 #undef HANDLE_TYPE_NAME_SYCL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 #undef HANDLE_TYPE_NAME_CPU
 #undef HANDLE_TYPE_NAME_GPU
@@ -505,127 +508,73 @@ REGISTER_KERNEL_BUILDER(Name("Tile")
                             .Device(DEVICE_CPU)
                             .HostMemory("multiples")
                             .TypeConstraint("Tmultiples"),
-                        TileOp);
+                        TileOp);
+REGISTER_KERNEL_BUILDER(Name("Tile")
+                            .Device(DEVICE_CPU)
+                            .HostMemory("multiples")
+                            .TypeConstraint("Tmultiples"),
+                        TileOp);
 REGISTER_KERNEL_BUILDER(
     Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"),
     TileGradientOp);
 
 #if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-
+#define REGISTER_GPU(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint("T")           \
+                              .TypeConstraint("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp);               \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint("T")           \
+                              .TypeConstraint("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp);               \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint("T")           \
+                              .TypeConstraint("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp);
+
+TF_CALL_float(REGISTER_GPU);
+TF_CALL_double(REGISTER_GPU);
+TF_CALL_half(REGISTER_GPU);
+TF_CALL_int16(REGISTER_GPU);
+TF_CALL_int32(REGISTER_GPU);
+TF_CALL_complex64(REGISTER_GPU);
+TF_CALL_complex128(REGISTER_GPU)
+
+#undef REGISTER_GPU
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp);
-
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint("T")
-                            .TypeConstraint("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp);
-#endif // TENSORFLOW_USE_SYCL
+#define REGISTER_SYCL(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint("T")           \
+                              .TypeConstraint("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp);              \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint("T")           \
+                              .TypeConstraint("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp);              \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint("T")           \
+                              .TypeConstraint("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp);
+
+    TF_CALL_float(REGISTER_SYCL);
+TF_CALL_double(REGISTER_SYCL);
+
+#undef REGISTER_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc
index e151b38d90..20f0edf309 100644
--- a/tensorflow/core/kernels/transpose_op.cc
+++ b/tensorflow/core/kernels/transpose_op.cc
@@ -91,6 +91,26 @@ REGISTER_KERNEL_BUILDER(Name("InvertPermutation")
                         InvertPermutationOp);
 #endif  // TENSORFLOW_USE_SYCL
 
+namespace {
+template 
+Status PermutationHelper(const Tensor& perm, const int dims,
+                         std::vector* permutation) {
+  auto Vperm = perm.vec();
+  if (dims != Vperm.size()) {
+    return errors::InvalidArgument("transpose expects a vector of size ", dims,
+                                   ". But input(1) is a vector of size ",
+                                   Vperm.size());
+  }
+  // using volatile instead of SubtleMustCopy here so that the
+  // asynchrony boundary is permutation.
+  const volatile Tperm* perm_begin =
+      reinterpret_cast(Vperm.data());
+  *permutation = std::vector(perm_begin, perm_begin + dims);
+
+  return Status::OK();
+}
+}  // namespace
+
 // output = TransposeOp(T input, T perm) takes a tensor
 // of type T and rank N, and a permutation of 0, 1, ..., N-1. It
 // shuffles the dimensions of the input tensor according to permutation.
@@ -113,17 +133,16 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
   OP_REQUIRES(ctx, TensorShapeUtils::IsVector(perm.shape()),
               errors::InvalidArgument("perm must be a vector, not ",
                                       perm.shape().DebugString()));
-  auto Vperm = perm.vec();
+
+  // Although Tperm may be an int64 type, an int32 is sufficient to hold
+  // dimension range values, so the narrowing here should be safe.
+  std::vector permutation;
   const int dims = input.dims();
-  OP_REQUIRES(ctx, dims == Vperm.size(),
-              errors::InvalidArgument(
-                  "transpose expects a vector of size ", input.dims(),
-                  ". But input(1) is a vector of size ", Vperm.size()));
-  // using volatile instead of SubtleMustCopy here so that the
-  // asynchrony boundary is permutation.
-  const volatile int32* perm_begin =
-      reinterpret_cast(Vperm.data());
-  const std::vector permutation(perm_begin, perm_begin + dims);
+  if (perm.dtype() == DT_INT32) {
+    OP_REQUIRES_OK(ctx, PermutationHelper(perm, dims, &permutation));
+  } else {
+    OP_REQUIRES_OK(ctx, PermutationHelper(perm, dims, &permutation));
+  }
   TensorShape shape;
 
   // Check whether permutation is a permutation of integers of [0 .. dims).
@@ -142,10 +161,9 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
     }
   }
   for (int i = 0; i < dims; ++i) {
-    OP_REQUIRES(
-        ctx, bits[i],
-        errors::InvalidArgument(i, " is missing from {",
-                                str_util::Join(permutation, ","), "}."));
+    OP_REQUIRES(ctx, bits[i], errors::InvalidArgument(
+                                  i, " is missing from {",
+                                  str_util::Join(permutation, ","), "}."));
   }
 
   // 0-D, 1-D, and identity transposes do nothing.
@@ -185,18 +203,16 @@ Status ConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
 }
 
 #ifdef INTEL_MKL
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
-                          MklTransposeCpuOp);                 \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
+                          MklTransposeCpuOp);         \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
                           MklConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER);
 REGISTER(bfloat16);
@@ -204,18 +220,16 @@ REGISTER(bfloat16);
 
 #else  // INTEL_MKL
 
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeCpuOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
+                          TransposeCpuOp);            \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER)
 REGISTER(bfloat16);
@@ -238,18 +252,16 @@ Status ConjugateTransposeGpuOp::DoTranspose(OpKernelContext* ctx,
                                             perm, out);
 }
 
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_GPU)             \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeGpuOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_GPU)             \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_GPU)     \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
+                          TransposeGpuOp);            \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_GPU)     \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeGpuOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
@@ -270,18 +282,16 @@ Status ConjugateTransposeSyclOp::DoTranspose(OpKernelContext* ctx,
   return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), in,
                                             perm, out);
 }
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_SYCL)            \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeSyclOp);                   \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_SYCL)            \
-                              .TypeConstraint("T")         \
-                              .TypeConstraint("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_SYCL)    \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
+                          TransposeSyclOp);           \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_SYCL)    \
+                              .TypeConstraint("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeSyclOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 14b87f0edf..c5935141f8 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -739,7 +739,7 @@ REGISTER_OP("Diag")
     .Attr("T: {float, double, int32, int64, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle in = c->input(0);
-      TF_RETURN_IF_ERROR(c->WithRankAtMost(in, 3, &in));
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(in, 1, &in));
       // Output shape is original concatenated with itself.
       ShapeHandle out;
       TF_RETURN_IF_ERROR(c->Concatenate(in, in, &out));
@@ -767,7 +767,7 @@ tf.diag(diagonal) ==> [[1, 0, 0, 0]
                        [0, 0, 0, 4]]
 ```
 
-diagonal: Rank k tensor where k is at most 3.
+diagonal: Rank k tensor where k is at most 1.
 )doc");
 
 // --------------------------------------------------------------------------
@@ -783,9 +783,9 @@ REGISTER_OP("DiagPart")
       }
       // Rank must be even, and result will have rank .
       const int32 rank = c->Rank(in);
-      if ((rank % 2) != 0 || rank > 6) {
+      if ((rank % 2) != 0 || rank <= 0) {
         return errors::InvalidArgument(
-            "Input must have even rank <= 6, input rank is ", rank);
+            "Input must have even and non-zero rank, input rank is ", rank);
       }
       const int32 mid = rank / 2;
 
@@ -820,7 +820,7 @@ For example:
 tf.diag_part(input) ==> [1, 2, 3, 4]
 ```
 
-input: Rank k tensor where k is 2, 4, or 6.
+input: Rank k tensor where k is even and not zero.
 diagonal: The extracted diagonal.
 
 )doc");
@@ -1175,7 +1175,7 @@ For example:
 #                  [20, 21, 22, 23]]]]
 # tensor 't' shape is [1, 2, 3, 4]
 
-# 'dims' is [3] or 'dims' is -1
+# 'dims' is [3] or 'dims' is [-1]
 reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
                         [ 7,  6,  5,  4],
                         [ 11, 10, 9, 8]],
@@ -2283,6 +2283,8 @@ size(t) ==> 12
 
 namespace {
 
+// This SliceHelper processes the output shape of the `slice`
+// when the tensor of `sizes` is available.
 template 
 Status SliceHelper(InferenceContext* c, ShapeHandle begin_value,
                    const Tensor* sizes_value,
@@ -2308,7 +2310,6 @@ Status SliceHelper(InferenceContext* c, ShapeHandle begin_value,
 
   return Status::OK();
 }
-
 }  // namespace
 
 // --------------------------------------------------------------------------
@@ -2339,9 +2340,10 @@ REGISTER_OP("Slice")
       ShapeHandle begin_value;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &begin_value));
 
-      // NOTE(mrry): We can't use `MakeShapeFromShapeTensor` for `sizes` because
-      // it might contain -1, which can't be represented (-1 in the ShapeHandle
-      // would mean "unknown".
+      // We check the tensor value here and will only use
+      // `MakeShapeFromShapeTensor` when `sizes_value` is null.
+      // The reason is that `sizes`might contain -1, which can't
+      // be represented (-1 in the ShapeHandle would mean "unknown".
       const Tensor* sizes_value = c->input_tensor(2);
 
       if (sizes_value != nullptr) {
@@ -2361,6 +2363,28 @@ REGISTER_OP("Slice")
         c->set_output(0, c->MakeShape(dims));
         return Status::OK();
       } else {
+        // In case `sizes` is not available (`sizes_value` is null),
+        // we could try to use `MakeShapeFromShapeTensor` here.
+        // If sizes contain -1, we will simply consider it as `Unknown`.
+        // This is less than ideal but still an improvement of shape inference.
+        // The following is an example that returns [None, 1, None] with this
+        // code path:
+        //   z = tf.zeros((1, 2, 3))
+        //   m = tf.slice(z, [0, 0, 0], [tf.constant(1) + 0, 1, -1])
+        //   m.get_shape().as_list()
+        ShapeHandle sizes_value;
+        TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &sizes_value));
+        if (c->RankKnown(sizes_value)) {
+          TF_RETURN_IF_ERROR(
+              c->WithRank(begin_value, c->Rank(sizes_value), &begin_value));
+          std::vector dims;
+          for (int i = 0; i < c->Rank(sizes_value); ++i) {
+            dims.emplace_back(c->Dim(sizes_value, i));
+          }
+          c->set_output(0, c->MakeShape(dims));
+          return Status::OK();
+        }
+
         // We might know the rank of the input.
         if (c->RankKnown(input)) {
           c->set_output(0, c->UnknownShapeOfRank(c->Rank(input)));
diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc
index a5d7a32e05..94eb120175 100644
--- a/tensorflow/core/ops/array_ops_test.cc
+++ b/tensorflow/core/ops/array_ops_test.cc
@@ -186,21 +186,20 @@ TEST(ArrayOpsTest, Identity_ShapeFnHandles) {
 TEST(ArrayOpsTest, Diag_ShapeFn) {
   ShapeInferenceTestOp op("Diag");
   INFER_OK(op, "?", "?");
-  INFER_OK(op, "[]", "[]");
   INFER_OK(op, "[1,?,3]", "[d0_0,d0_1,d0_2,d0_0,d0_1,d0_2]");
-  INFER_ERROR("Shape must be at most rank 3 but is rank 4", op, "[?,1,2,3]");
+  INFER_OK(op, "[?,1,2,3]", "[d0_0,d0_1,d0_2,d0_3,d0_0,d0_1,d0_2,d0_3]");
+  INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[]");
 }
 
 TEST(ArrayOpsTest, DiagPart_ShapeFn) {
   ShapeInferenceTestOp op("DiagPart");
   INFER_OK(op, "?", "?");
-  INFER_OK(op, "[]", "[]");
   INFER_OK(op, "[1,?,?,4]", "[d0_0,d0_3]");
   INFER_OK(op, "[1,?,3,?,4,3]", "[d0_0,d0_4,d0_2|d0_5]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 1", op, "[?]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 3", op, "[1,2,3]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 8", op,
-              "[1,2,3,?,?,?,?,?]");
+  INFER_OK(op, "[1,2,3,?,?,?,?,4]", "[d0_0,d0_1,d0_2,d0_7]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[?]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[1,2,3]");
   INFER_ERROR("Dimensions must be equal, but are 2 and 10", op, "[1,2,?,10]");
 }
 
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index a44bac60bf..e9bf29d172 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -151,7 +151,7 @@ REGISTER_OP("ResizeArea")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -179,7 +179,7 @@ REGISTER_OP("ResizeBicubic")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -227,7 +227,7 @@ REGISTER_OP("ResizeBilinear")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -311,7 +311,7 @@ REGISTER_OP("ResizeNearestNeighbor")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: T")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -453,7 +453,36 @@ REGISTER_OP("DecodeAndCropJpeg")
     .Attr("acceptable_fraction: float = 1.0")
     .Attr("dct_method: string = ''")
     .Output("image: uint8")
-    .SetShapeFn(DecodeImageShapeFn)
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
+      DimensionHandle channels_dim = c->UnknownDim();
+      DimensionHandle h = c->UnknownDim();
+      DimensionHandle w = c->UnknownDim();
+
+      int32 channels;
+      TF_RETURN_IF_ERROR(c->GetAttr("channels", &channels));
+      if (channels != 0) {
+        if (channels < 0) {
+          return errors::InvalidArgument("channels must be non-negative, got ",
+                                         channels);
+        }
+        channels_dim = c->MakeDim(channels);
+      }
+
+      DimensionHandle unused_dim;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithValue(c->Dim(unused, 0), 4, &unused_dim));
+
+      const Tensor* crop_window = c->input_tensor(1);
+      if (crop_window != nullptr) {
+        auto crop_window_vec = crop_window->vec();
+        h = c->MakeDim(crop_window_vec(2));
+        w = c->MakeDim(crop_window_vec(3));
+      }
+      c->set_output(0, c->MakeShape({h, w, channels_dim}));
+      return Status::OK();
+    })
     .Doc(strings::StrCat(R"doc(
 Decode and Crop a JPEG-encoded image to a uint8 tensor.
 )doc",
@@ -1068,7 +1097,7 @@ REGISTER_OP("CropAndResize")
     .Input("box_ind: int32")
     .Input("crop_size: int32")
     .Output("crops: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}")
     .Attr("method: {'bilinear'} = 'bilinear'")
     .Attr("extrapolation_value: float = 0")
     .SetShapeFn([](InferenceContext* c) {
@@ -1175,7 +1204,7 @@ REGISTER_OP("CropAndResizeGradBoxes")
     .Input("boxes: float")
     .Input("box_ind: int32")
     .Output("output: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}")
     .Attr("method: {'bilinear'} = 'bilinear'")
     .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->input(2));
diff --git a/tensorflow/core/ops/image_ops_test.cc b/tensorflow/core/ops/image_ops_test.cc
index c34b11a15e..5f0b391b0d 100644
--- a/tensorflow/core/ops/image_ops_test.cc
+++ b/tensorflow/core/ops/image_ops_test.cc
@@ -105,7 +105,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) {
                    .Input({"img", 0, DT_STRING})
                    .Input({"crop_window", 1, DT_INT32})
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,?]");
+  INFER_OK(op, "[];[?]", "[?,?,?]");
 
   // Set the channel, so that part of output shape is known.
   TF_ASSERT_OK(NodeDefBuilder("test", op_name)
@@ -113,7 +113,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) {
                    .Input({"crop_window", 1, DT_INT32})
                    .Attr("channels", 4)
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,4]");
+  INFER_OK(op, "[];[?]", "[?,?,4]");
 
   // Negative channel value is rejected.
   TF_ASSERT_OK(NodeDefBuilder("test", op_name)
@@ -139,7 +139,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_InvalidCropWindow) {
                    .Input({"img", 0, DT_STRING})
                    .Input({"crop_window", 1, DT_INT32})
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,?]");
+  INFER_OK(op, "[];[?]", "[?,?,?]");
 }
 
 TEST(ImageOpsTest, EncodeImage_ShapeFn) {
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index ab0bc258f7..61db896c51 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -49,6 +49,38 @@ inputs: Must all be the same size and shape.
 
 // --------------------------------------------------------------------------
 
+// Note that the following operator is just a placeholder and has no
+// associated kernel. The code in accumulate_n_optimizer.cc replaces
+// this placeholder with a graph of operators that do have kernels.
+// The Python code that generates instances of this op is currently in
+// contrib/framework/python/ops/accumulate_n_v2.py
+REGISTER_OP("AccumulateNV2")
+    .Input("inputs: N * T")
+    .Output("sum: T")
+    .Attr("N: int >= 1")
+    .Attr("T: numbertype")
+    .Attr("shape: shape")
+    .SetIsCommutative()
+    .SetIsAggregate()
+    .SetShapeFn(shape_inference::ExplicitShape)
+    .Doc(R"doc(
+Returns the element-wise sum of a list of tensors.
+
+`tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+wait for all of its inputs to be ready before beginning to sum. This can
+save memory if inputs are ready at different times, since minimum temporary
+storage is proportional to the output size rather than the inputs size.
+
+Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+
+Returns a `Tensor` of same shape and type as the elements of `inputs`.
+
+inputs: A list of `Tensor` objects, each with same shape and type.
+shape: Shape of elements of `inputs`.
+)doc");
+
+// --------------------------------------------------------------------------
+
 REGISTER_OP("BatchMatMul")
     .Input("x: T")
     .Input("y: T")
@@ -591,7 +623,7 @@ REGISTER_OP("TruncateDiv")
 Returns x / y element-wise for integer types.
 
 Truncation designates that negative numbers will round fractional quantities
-toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different
+toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
 than Python semantics. See `FloorDiv` for a division function that matches
 Python Semantics.
 
@@ -2218,6 +2250,44 @@ product: Pairwise cross product of the vectors in `a` and `b`.
 
 // --------------------------------------------------------------------------
 
+REGISTER_OP("HistogramFixedWidth")
+    .Input("values: T")
+    .Input("value_range: T")
+    .Input("nbins: int32")
+    .Output("out: dtype")
+    .Attr("T: {int32, int64, float32, float64}")
+    .Attr("dtype: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      c->set_output(0, c->UnknownShapeOfRank(1));
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Return histogram of values.
+
+Given the tensor `values`, this operation returns a rank 1 histogram counting
+the number of entries in `values` that fall into every bin.  The bins are
+equal width and determined by the arguments `value_range` and `nbins`.
+
+```python
+# Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)
+nbins = 5
+value_range = [0.0, 5.0]
+new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
+
+with tf.get_default_session() as sess:
+  hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)
+  variables.global_variables_initializer().run()
+  sess.run(hist) => [2, 1, 1, 0, 2]
+```
+
+values:  Numeric `Tensor`.
+value_range:  Shape [2] `Tensor` of same `dtype` as `values`.
+  values <= value_range[0] will be mapped to hist[0],
+  values >= value_range[1] will be mapped to hist[-1].
+nbins:  Scalar `int32 Tensor`.  Number of histogram bins.
+out: A 1-D `Tensor` holding histogram of values.
+)doc");
+
 REGISTER_OP("Bincount")
     .Input("arr: int32")
     .Input("size: int32")
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 5efa55b496..1d26660a4b 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -2260,6 +2260,56 @@ indices: The indices of `values` within the last dimension of `input`.
 
 // --------------------------------------------------------------------------
 
+REGISTER_OP("NthElement")
+    .Input("input: T")
+    .Input("n: int32")
+    .Output("values: T")
+    .Attr("reverse: bool = false")
+    .Attr("T: realnumbertype")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle input;
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input));
+
+      // Get the n value from input tensor, and make sure which is a scalar.
+      DimensionHandle n_dim;
+      TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(1, &n_dim));
+
+      // The last dimension of input tensor must be greater than N.
+      DimensionHandle last_dim = c->Dim(input, -1);
+      if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) &&
+          c->Value(last_dim) <= c->Value(n_dim)) {
+        return errors::InvalidArgument(
+            "Input must have last dimension > n = ", c->Value(n_dim), " but is ",
+            c->Value(last_dim));
+      }
+
+      // Reduce last_dim for output tensor
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->Subshape(input, 0, -1, &s));
+      c->set_output(0, s);
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Finds values of the `n`-th order statistic for the last dmension.
+
+If the input is a vector (rank-1), finds the entries which is the nth-smallest
+value in the vector and outputs their values as scalar tensor.
+
+For matrices (resp. higher rank input), computes the entries which is the
+nth-smallest value in each row (resp. vector along the last dimension). Thus,
+
+    values.shape = input.shape[:-1]
+
+input: 1-D or higher with last dimension at least `n+1`.
+n: 0-D. Position of sorted vector to select along the last dimension (along
+  each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+reverse: When set to True, find the nth-largest value in the vector and vice
+  versa.
+values: The `n`-th order statistic along each last dimensional slice.
+)doc");
+
+// --------------------------------------------------------------------------
+
 REGISTER_OP("FractionalMaxPool")
     .Input("value: T")
     .Output("output: T")
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 4628b725f8..94ecf4d5db 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -81,6 +81,30 @@ TEST(NNOpsTest, TopKV2_ShapeFn) {
       op, "[1,2,3,4];[]");
 }
 
+TEST(NNOpsTest, NthElement_ShapeFn) {
+  ShapeInferenceTestOp op("NthElement");
+  op.input_tensors.resize(2);
+
+  Tensor n_t;
+  op.input_tensors[1] = &n_t;
+  n_t = test::AsScalar(20);
+
+  INFER_OK(op, "?;[]", "?");
+  INFER_OK(op, "[21];[]", "[]");
+  INFER_OK(op, "[2,?,?];[]", "[d0_0,d0_1]");
+  INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]");
+
+  INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 1", op,
+              "[1];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 20", op,
+              "[1,2,3,20];[]");
+  n_t = test::AsScalar(-1);
+  INFER_ERROR(
+     "Dimension size, given by scalar input 1, must be non-negative but is -1",
+     op, "[1,2,3,4];[]");
+}
+
 TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) {
   ShapeInferenceTestOp op("BatchNormWithGlobalNormalization");
 
diff --git a/tensorflow/core/platform/s3/s3_crypto.cc b/tensorflow/core/platform/s3/s3_crypto.cc
index 14bbed19a5..d7062a59d2 100644
--- a/tensorflow/core/platform/s3/s3_crypto.cc
+++ b/tensorflow/core/platform/s3/s3_crypto.cc
@@ -71,7 +71,7 @@ class S3Sha256OpenSSLImpl : public Aws::Utils::Crypto::Hash {
     SHA256_Init(&sha256);
 
     auto currentPos = stream.tellg();
-    if (currentPos == -1) {
+    if (currentPos == std::streampos(std::streamoff(-1))) {
       currentPos = 0;
       stream.clear();
     }
diff --git a/tensorflow/core/profiler/README.md b/tensorflow/core/profiler/README.md
index 92bce9c1ce..8ca26fa5dc 100644
--- a/tensorflow/core/profiler/README.md
+++ b/tensorflow/core/profiler/README.md
@@ -48,7 +48,7 @@ bazel-bin/tensorflow/python/profiler/profiler_ui \
 # Create options to profile the time and memory information.
 builder = tf.profiler.ProfileOptionBuilder
 opts = builder(builder.time_and_memory()).order_by('micros').build()
-# Create a profiling context, set contructor argument `trace_steps`, 
+# Create a profiling context, set constructor argument `trace_steps`, 
 # `dump_steps` to empty for explicit control.
 with tf.contrib.tfprof.ProfileContext('/tmp/train_dir',
                                       trace_steps=[],
diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md
index ddee63ad42..4c73e372e3 100644
--- a/tensorflow/core/profiler/g3doc/options.md
+++ b/tensorflow/core/profiler/g3doc/options.md
@@ -43,7 +43,7 @@ In graph view, in means the number of hops in the graph.
 
 ### Times
 
-Most machines have mutli-core CPUs. Some installs one or more accelerators.
+Most machines have multi-core CPUs. Some installs one or more accelerators.
 Each accelerator usually performs massive parallel processing. The profiler
 tracks the accumulated processing times. Hence, the accumulated processing
 time is likely larger than the time of each step.
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index ccb861c93a..5d2298f7b7 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 3
+#define TF_MINOR_VERSION 4
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc0"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md
index 7609ca91d0..b3ebaa0f0a 100644
--- a/tensorflow/docs_src/api_guides/python/reading_data.md
+++ b/tensorflow/docs_src/api_guides/python/reading_data.md
@@ -67,7 +67,7 @@ A typical queue-based pipeline for reading records from files has the following
 8.  Example queue
 
 Warning: This section discusses implementing input pipelines using the
-queue-based APIs which can be cleanly replaced by the @{$datasets$Dataset API}.
+queue-based APIs which can be cleanly replaced by the @{$datasets$Datasets API}.
 
 ### Filenames, shuffling, and epoch limits
 
diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md
index ab270d1408..790de6679b 100644
--- a/tensorflow/docs_src/get_started/estimator.md
+++ b/tensorflow/docs_src/get_started/estimator.md
@@ -28,7 +28,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import urllib
+from six.moves.urllib.request import urlopen
 
 import numpy as np
 import tensorflow as tf
@@ -44,13 +44,13 @@ IRIS_TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
 def main():
   # If the training and test sets aren't stored locally, download them.
   if not os.path.exists(IRIS_TRAINING):
-    raw = urllib.urlopen(IRIS_TRAINING_URL).read()
-    with open(IRIS_TRAINING, "w") as f:
+    raw = urlopen(IRIS_TRAINING_URL).read()
+    with open(IRIS_TRAINING, "wb") as f:
       f.write(raw)
 
   if not os.path.exists(IRIS_TEST):
-    raw = urllib.urlopen(IRIS_TEST_URL).read()
-    with open(IRIS_TEST, "w") as f:
+    raw = urlopen(IRIS_TEST_URL).read()
+    with open(IRIS_TEST, "wb") as f:
       f.write(raw)
 
   # Load datasets.
@@ -167,7 +167,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import urllib
+from six.moves.urllib.request import urlopen
 
 import tensorflow as tf
 import numpy as np
@@ -184,13 +184,13 @@ them.
 
 ```python
 if not os.path.exists(IRIS_TRAINING):
-  raw = urllib.urlopen(IRIS_TRAINING_URL).read()
-  with open(IRIS_TRAINING,'w') as f:
+  raw = urlopen(IRIS_TRAINING_URL).read()
+  with open(IRIS_TRAINING,'wb') as f:
     f.write(raw)
 
 if not os.path.exists(IRIS_TEST):
-  raw = urllib.urlopen(IRIS_TEST_URL).read()
-  with open(IRIS_TEST,'w') as f:
+  raw = urlopen(IRIS_TEST_URL).read()
+  with open(IRIS_TEST,'wb') as f:
     f.write(raw)
 ```
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 7ebf5c4a2c..586bb6dead 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -35,7 +35,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for Mac OS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index b991fd0f93..1d00661d83 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -35,7 +35,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 2adcd4da73..3b3acfdcb3 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -34,7 +34,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 
   org.tensorflow
   tensorflow
-  1.3.0
+  1.4.0-rc0
 
 ```
 
@@ -63,7 +63,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                
                  org.tensorflow
                  tensorflow
-                 1.3.0
+                 1.4.0-rc0
                
              
          
@@ -122,7 +122,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or Mac OS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.3.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -141,7 +141,7 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -149,10 +149,10 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.3.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.3.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc0.zip).
   3. Extract this .zip file.
 
 
@@ -200,7 +200,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-
javac -cp libtensorflow-1.3.0.jar HelloTF.java

+
javac -cp libtensorflow-1.4.0-rc0.jar HelloTF.java

 
 
 ### Running
@@ -214,11 +214,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and Mac OS X:
 
-
java -cp libtensorflow-1.3.0.jar:. -Djava.library.path=./jni HelloTF

+
java -cp libtensorflow-1.4.0-rc0.jar:. -Djava.library.path=./jni HelloTF

 
 And the following command line executes the `HelloTF` program on Windows:
 
-
java -cp libtensorflow-1.3.0.jar;. -Djava.library.path=jni HelloTF

+
java -cp libtensorflow-1.4.0-rc0.jar;. -Djava.library.path=jni HelloTF

 
 If the program prints Hello from version, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 576099f054..9d204cc246 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -42,8 +42,20 @@ must be installed on your system:
     a list of supported GPU cards.
   * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface.
     This library provides advanced profiling support. To install this library,
-    issue the following command:
+    issue the following command for CUDA Toolkit >= 8.0:
 
+    
+    $ sudo apt-get install cuda-command-line-tools
+    

+    
+    and add its path to your `LD_LIBRARY_PATH` environment variable:
+
+    
 
+    $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64 
+    

+
+    For CUDA Toolkit <= 7.5 do:
+    
     
     $ sudo apt-get install libcupti-dev
     

@@ -172,7 +184,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      virtualenv environment:
 
      
(tensorflow)$ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl

+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl

 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -277,7 +289,7 @@ take the following steps:
 
      
      $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
      

 
      If this step fails, see
@@ -445,7 +457,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named tensorflow to run a version
      of Python by invoking the following command:
 
-     
$ conda create -n tensorflow python=2.7 # or python=3.3, etc.

+     
$ conda create -n tensorflow pip python=2.7 # or python=3.3, etc.

 
   3. Activate the conda environment by issuing the following command:
 
@@ -464,7 +476,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl

+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl

 
 
 
@@ -632,14 +644,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp27-none-linux_x86_64.whl
 

 
 
 GPU support:
 
 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp27-none-linux_x86_64.whl
 

 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -651,14 +663,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
 

 
 
 GPU support:
 
 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
 

 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -670,14 +682,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
 

 
 
 GPU support:
 
 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
 

 
 
@@ -689,14 +701,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
 

 
 
 GPU support:
 
 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
 

 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index b6daeb0dd6..6da22784bf 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -109,7 +109,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl

+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -230,7 +230,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -321,7 +321,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 2. Create a conda environment named `tensorflow` by invoking the following command: - 
$ conda create -n tensorflow python=2.7 # or python=3.3, etc.
+ 
$ conda create -n tensorflow pip python=2.7 # or python=3.3, etc.
3. Activate the conda environment by issuing the following command: @@ -339,7 +339,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl @@ -512,7 +512,7 @@ This section documents the relevant values for Mac OS installations. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
@@ -520,7 +520,7 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index e6a4088656..b853d87816 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -137,8 +137,15 @@ The following NVIDIA software must be installed on your system: particularly the description of appending the appropriate pathname to your `LD_LIBRARY_PATH` environment variable. -Finally, you must also install `libcupti-dev` by invoking the following -command: +Finally, you must also install `libcupti` which for Cuda Toolkit >= 8.0 you do via + +
 $ sudo apt-get install cuda-command-line-tools
+ +and add its path to your `LD_LIBRARY_PATH` environment variable: + +
 $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64
+ +For Cuda Toolkit <= 7.5, you install `libcupti-dev` by invoking the following command: 
 $ sudo apt-get install libcupti-dev
@@ -342,10 +349,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.4.0dev on Linux: +for TensorFlow 1.4.0rc0 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0dev-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc0-py2-none-any.whl
## Validate your installation @@ -434,8 +441,8 @@ Stack Overflow and specify the `tensorflow` tag. **Linux** - - + + @@ -447,7 +454,7 @@ Stack Overflow and specify the `tensorflow` tag. **Mac**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.3.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.3.0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.568
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.4.0rc0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.568
tensorflow-1.2.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.2.0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.55.18
tensorflow-1.1.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.2N/AN/A
- + @@ -458,8 +465,8 @@ Stack Overflow and specify the `tensorflow` tag. **Windows**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.3.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
ttensorflow-1.2.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
ttensorflow-1.1.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
ttensorflow_gpu-1.1.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
- - + + diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index ae8749c231..f0d580d803 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -105,7 +105,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 2. Create a conda environment named tensorflow by invoking the following command: - 
C:\> conda create -n tensorflow python=3.5
+ 
C:\> conda create -n tensorflow pip python=3.5
3. Activate the conda environment by issuing the following command: diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md index 06bb40f64d..da556bd848 100644 --- a/tensorflow/docs_src/performance/performance_guide.md +++ b/tensorflow/docs_src/performance/performance_guide.md @@ -127,7 +127,7 @@ Reading large numbers of small files significantly impacts I/O performance. One approach to get maximum I/O throughput is to preprocess input data into larger (~100MB) `TFRecord` files. For smaller data sets (200MB-1GB), the best approach is often to load the entire data set into memory. The document -[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/slim#Data) +[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/research/slim#Data) includes information and scripts for creating `TFRecords` and this [script](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py) converts the CIFAR-10 data set into `TFRecords`. diff --git a/tensorflow/docs_src/performance/performance_models.md b/tensorflow/docs_src/performance/performance_models.md index 183bbc75a9..fcda19e74c 100644 --- a/tensorflow/docs_src/performance/performance_models.md +++ b/tensorflow/docs_src/performance/performance_models.md @@ -345,7 +345,7 @@ executing the main script * **`num_gpus`**: Number of GPUs to use. * **`data_dir`**: Path to data to process. If not set, synthetic data is used. To use Imagenet data use these - [instructions](https://github.com/tensorflow/models/tree/master/inception#getting-started) + [instructions](https://github.com/tensorflow/models/tree/master/research/inception#getting-started) as a starting point. * **`batch_size`**: Batch size for each GPU. * **`variable_update`**: The method for managing variables: `parameter_server` diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md index 38e5612fb4..f458cbcef2 100644 --- a/tensorflow/docs_src/programmers_guide/datasets.md +++ b/tensorflow/docs_src/programmers_guide/datasets.md @@ -44,7 +44,7 @@ To start an input pipeline, you must define a *source*. For example, to construct a `Dataset` from some tensors in memory, you can use `tf.data.Dataset.from_tensors()` or `tf.data.Dataset.from_tensor_slices()`. Alternatively, if your input -data are on disk in the recommend TFRecord format, you can construct a +data are on disk in the recommended TFRecord format, you can construct a `tf.data.TFRecordDataset`. Once you have a `Dataset` object, you can *transform* it into a new `Dataset` by diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 6ba8bb7a34..10f53fe8f2 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -404,8 +404,8 @@ y = tf.square(x) with tf.Session() as sess: # Feeding a value changes the result that is returned when you evaluate `y`. - print(sess.run(y, {x: [1.0, 2.0, 3.0]}) # => "[1.0, 4.0, 9.0]" - print(sess.run(y, {x: [0.0, 0.0, 5.0]}) # => "[0.0, 0.0, 25.0]" + print(sess.run(y, {x: [1.0, 2.0, 3.0]})) # => "[1.0, 4.0, 9.0]" + print(sess.run(y, {x: [0.0, 0.0, 5.0]})) # => "[0.0, 0.0, 25.0]" # Raises `tf.errors.InvalidArgumentError`, because you must feed a value for # a `tf.placeholder()` when evaluating a tensor that depends on it. diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md index 9262143ad8..6bc2cbb9e3 100644 --- a/tensorflow/docs_src/programmers_guide/saved_model.md +++ b/tensorflow/docs_src/programmers_guide/saved_model.md @@ -158,6 +158,39 @@ Notes: optionally choose names for the variables in the checkpoint files. +### Inspect variables in a checkpoint + +We can quickly inspect variables in a checkpoint with the +[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library. + +Continuing from the save/restore examples shown earlier: + +```python +# import the inspect_checkpoint library +from tensorflow.python.tools import inspect_checkpoint as chkp + +# print all tensors in checkpoint file +chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='', all_tensors=True) + +# tensor_name: v1 +# [ 1. 1. 1.] +# tensor_name: v2 +# [-1. -1. -1. -1. -1.] + +# print only tensor v1 in checkpoint file +chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v1', all_tensors=False) + +# tensor_name: v1 +# [ 1. 1. 1.] + +# print only tensor v2 in checkpoint file +chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v2', all_tensors=False) + +# tensor_name: v2 +# [-1. -1. -1. -1. -1.] +``` + + ## Overview of saving and restoring models diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md index 3055c54021..6292c1a01e 100644 --- a/tensorflow/docs_src/tutorials/wide.md +++ b/tensorflow/docs_src/tutorials/wide.md @@ -426,8 +426,7 @@ m = tf.estimator.LinearClassifier( optimizer=tf.train.FtrlOptimizer( learning_rate=0.1, l1_regularization_strength=1.0, - l2_regularization_strength=1.0), - model_dir=model_dir) + l2_regularization_strength=1.0)) ``` One important difference between L1 and L2 regularization is that L1 diff --git a/tensorflow/examples/get_started/regression/imports85.py b/tensorflow/examples/get_started/regression/imports85.py index 96a464920a..6bee556eb8 100644 --- a/tensorflow/examples/get_started/regression/imports85.py +++ b/tensorflow/examples/get_started/regression/imports85.py @@ -127,7 +127,7 @@ def dataset(y_name="price", train_fraction=0.7): def in_test_set(line): """Returns a boolean tensor, true if the line is in the training set.""" # Items not in the training set are in the test set. - # This line must use `~` instead of `not` beacuse `not` only works on python + # This line must use `~` instead of `not` because `not` only works on python # booleans but we are dealing with symbolic tensors. return ~in_training_set(line) diff --git a/tensorflow/examples/get_started/regression/linear_regression_categorical.py b/tensorflow/examples/get_started/regression/linear_regression_categorical.py index 860d0e437c..e2ad415fbc 100644 --- a/tensorflow/examples/get_started/regression/linear_regression_categorical.py +++ b/tensorflow/examples/get_started/regression/linear_regression_categorical.py @@ -67,7 +67,7 @@ def main(argv): # The second way, appropriate for an unspecified vocabulary, is to create a # hashed column. It will create a fixed length list of weights, and - # automatically assign each input categort to a weight. Due to the + # automatically assign each input category to a weight. Due to the # pseudo-randomness of the process, some weights may be shared between # categories, while others will remain unused. make_column = tf.feature_column.categorical_column_with_hash_bucket( diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py index 33a09bb6e0..1e0966475b 100755 --- a/tensorflow/examples/learn/resnet.py +++ b/tensorflow/examples/learn/resnet.py @@ -190,8 +190,8 @@ def main(unused_args): # Calculate accuracy. test_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: mnist.train.images}, - y=mnist.train.labels.astype(np.int32), + x={X_FEATURE: mnist.test.images}, + y=mnist.test.labels.astype(np.int32), num_epochs=1, shuffle=False) scores = classifier.evaluate(input_fn=test_input_fn) diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py index 1fa2b14869..142e45a2e8 100644 --- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py +++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py @@ -115,11 +115,9 @@ def generate_batch(batch_size, num_skips, skip_window): data_index += span for i in range(batch_size // num_skips): context_words = [w for w in range(span) if w != skip_window] - random.shuffle(context_words) - words_to_use = collections.deque(context_words) - for j in range(num_skips): + words_to_use = random.sample(context_words, num_skips) + for j, context_word in enumerate(words_to_use): batch[i * num_skips + j] = buffer[skip_window] - context_word = words_to_use.pop() labels[i * num_skips + j, 0] = buffer[context_word] if data_index == len(data): buffer[:] = data[:span] diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index a380bc2c71..d74cb32c5a 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -24,6 +24,7 @@ java_library( ], data = [":libtensorflow_jni"], javacopts = JAVACOPTS, + plugins = [":processor"], visibility = ["//visibility:public"], ) @@ -41,6 +42,21 @@ filegroup( ], ) +java_plugin( + name = "processor", + generates_api = True, + processor_class = "org.tensorflow.processor.OperatorProcessor", + visibility = ["//visibility:public"], + deps = [":processor_library"], +) + +java_library( + name = "processor_library", + srcs = glob(["src/gen/java/org/tensorflow/processor/**/*.java"]), + javacopts = JAVACOPTS, + resources = glob(["src/gen/resources/META-INF/services/javax.annotation.processing.Processor"]), +) + filegroup( name = "java_op_sources", srcs = glob(["src/main/java/org/tensorflow/op/**/*.java"]) + [ @@ -264,6 +280,29 @@ tf_java_test( ], ) +#java_test( +# name = "OperatorProcessorTest", +# size = "small", +# srcs = ["src/test/java/org/tensorflow/processor/OperatorProcessorTest.java"], +# javacopts = JAVACOPTS, +# resources = [":processor_test_resources"], +# test_class = "org.tensorflow.processor.OperatorProcessorTest", +# deps = [ +# ":processor_library", +# "@junit", +# "@com_google_testing_compile", +# "@com_google_truth", +# ], +#) + +filegroup( + name = "processor_test_resources", + srcs = glob([ + "src/test/resources/org/tensorflow/**/*.java", + "src/main/java/org/tensorflow/op/annotation/Operator.java", + ]), +) + filegroup( name = "libtensorflow_jni", srcs = select({ diff --git a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java new file mode 100644 index 0000000000..45e42878c7 --- /dev/null +++ b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java @@ -0,0 +1,164 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.processor; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.Filer; +import javax.annotation.processing.Messager; +import javax.annotation.processing.ProcessingEnvironment; +import javax.annotation.processing.RoundEnvironment; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.Element; +import javax.lang.model.element.TypeElement; +import javax.tools.Diagnostic.Kind; + +/** + * A compile-time Processor that aggregates classes annotated with {@link + * org.tensorflow.op.annotation.Operator} and generates the {@code Ops} convenience API. Please + * refer to the {@link org.tensorflow.op.annotation.Operator} annotation for details about the API + * generated for each annotated class. + * + *

Note that this processor can only be invoked once, in a single compilation run that includes + * all the {@code Operator} annotated source classes. The reason is that the {@code Ops} API is an + * "aggregating" API, and annotation processing does not permit modifying an already generated + * class. + * + * @see org.tensorflow.op.annotation.Operator + */ +public final class OperatorProcessor extends AbstractProcessor { + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latestSupported(); + } + + @Override + public synchronized void init(ProcessingEnvironment processingEnv) { + super.init(processingEnv); + messager = processingEnv.getMessager(); + filer = processingEnv.getFiler(); + } + + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) { + // Nothing needs to be done at the end of all rounds. + if (roundEnv.processingOver()) { + return false; + } + + // Nothing to look at in this round. + if (annotations.size() == 0) { + return false; + } + + // We expect to be registered for exactly one annotation. + if (annotations.size() != 1) { + throw new IllegalStateException( + "Unexpected - multiple annotations registered: " + annotations); + } + TypeElement annotation = annotations.iterator().next(); + Set annotated = roundEnv.getElementsAnnotatedWith(annotation); + + // If there are no annotated elements, claim the annotion but do nothing. + if (annotated.size() == 0) { + return true; + } + + // This processor has to aggregate all op classes in one round, as it generates a single Ops + // API class which cannot be modified once generated. If we find an annotation after we've + // generated our code, flag the location of each such class. + if (hasRun) { + for (Element e : annotated) { + error( + e, + "The Operator processor has already processed @Operator annotated sources\n" + + "and written out an Ops API. It cannot process additional @Operator sources.\n" + + "One reason this can happen is if other annotation processors generate\n" + + "new @Operator source files."); + } + return true; + } + + // Collect all classes tagged with our annotation. + Set opClasses = new HashSet(); + if (!collectOpClasses(roundEnv, opClasses, annotation)) { + return true; + } + + // Nothing to do when there are no tagged classes. + if (opClasses.isEmpty()) { + return true; + } + + // TODO:(kbsriram) validate operator classes and generate Op API. + writeApi(); + hasRun = true; + return true; + } + + @Override + public Set getSupportedAnnotationTypes() { + return Collections.singleton(String.format("%s.annotation.Operator", OP_PACKAGE)); + } + + private void writeApi() { + // Generate an empty class for now and get the build working correctly. This will be changed to + // generate the actual API once we've done with build-related changes. + // TODO:(kbsriram) + try (PrintWriter writer = + new PrintWriter(filer.createSourceFile(String.format("%s.Ops", OP_PACKAGE)).openWriter())) { + writer.println(String.format("package %s;", OP_PACKAGE)); + writer.println("public class Ops{}"); + } catch (IOException e) { + error(null, "Unexpected failure generating API: %s", e.getMessage()); + } + } + + private boolean collectOpClasses( + RoundEnvironment roundEnv, Set opClasses, TypeElement annotation) { + boolean result = true; + for (Element e : roundEnv.getElementsAnnotatedWith(annotation)) { + // @Operator can only apply to types, so e must be a TypeElement. + if (!(e instanceof TypeElement)) { + error( + e, + "@Operator can only be applied to classes, but this is a %s", + e.getKind().toString()); + result = false; + continue; + } + opClasses.add((TypeElement) e); + } + return result; + } + + private void error(Element e, String message, Object... args) { + if (args != null && args.length > 0) { + message = String.format(message, args); + } + messager.printMessage(Kind.ERROR, message, e); + } + + private Filer filer; + private Messager messager; + private boolean hasRun = false; + private static final String OP_PACKAGE = "org.tensorflow.op"; +} diff --git a/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor b/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor new file mode 100644 index 0000000000..9a4fc98a89 --- /dev/null +++ b/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor @@ -0,0 +1 @@ +org.tensorflow.processor.OperatorProcessor diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java index 59476fb43d..3782240edb 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java +++ b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java @@ -54,7 +54,7 @@ import java.lang.annotation.Target; */ @Documented @Target(ElementType.TYPE) -@Retention(RetentionPolicy.CLASS) +@Retention(RetentionPolicy.SOURCE) public @interface Operator { /** * Specify an optional group within the {@code Ops} class. diff --git a/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java b/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java new file mode 100644 index 0000000000..9fa1bad20d --- /dev/null +++ b/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java @@ -0,0 +1,51 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.processor; + +import static com.google.testing.compile.CompilationSubject.assertThat; + +import com.google.testing.compile.Compilation; +import com.google.testing.compile.Compiler; +import com.google.testing.compile.JavaFileObjects; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Basic tests for {@link org.tensorflow.processor.operator.OperatorProcessor}. */ +@RunWith(JUnit4.class) +public final class OperatorProcessorTest { + + @Test + public void basicGood() { + Compilation compile = compile("org/tensorflow/processor/operator/good/BasicGood.java"); + assertThat(compile).succeededWithoutWarnings(); + assertThat(compile).generatedSourceFile("org.tensorflow.op.Ops"); + } + + @Test + public void basicBad() { + assertThat(compile("org/tensorflow/processor/operator/bad/BasicBad.java")).failed(); + } + + // Create a compilation unit that includes the @Operator annotation and processor. + private static Compilation compile(String path) { + return Compiler.javac() + .withProcessors(new OperatorProcessor()) + .compile( + JavaFileObjects.forResource("src/main/java/org/tensorflow/op/annotation/Operator.java"), + JavaFileObjects.forResource(path)); + } +} diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java new file mode 100644 index 0000000000..7d12857dfa --- /dev/null +++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java @@ -0,0 +1,22 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.processor.operator.bad; + +import org.tensorflow.op.annotation.Operator; + +public class BasicBad { + @Operator int foo; +} diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java new file mode 100644 index 0000000000..4cf175f00d --- /dev/null +++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java @@ -0,0 +1,21 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.processor.operator.good; + +import org.tensorflow.op.annotation.Operator; + +@Operator +public class BasicGood {} diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 4382eeb9a8..953aa566f0 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -4202,6 +4202,19 @@ cuda_py_test( main = "client/session_benchmark.py", ) +cuda_py_test( + name = "nn_grad_test", + size = "small", + srcs = ["ops/nn_grad_test.py"], + additional_deps = [ + ":client_testlib", + ":framework_for_generated_wrappers", + ":nn_grad", + ":nn_ops", + "//third_party/py/numpy", + ], +) + py_library( name = "tf_item", srcs = [ diff --git a/tensorflow/python/debug/cli/tensor_format.py b/tensorflow/python/debug/cli/tensor_format.py index 7a5597db12..05ccf93f15 100644 --- a/tensorflow/python/debug/cli/tensor_format.py +++ b/tensorflow/python/debug/cli/tensor_format.py @@ -480,7 +480,7 @@ def _pad_string_to_length(string, length): def numeric_summary(tensor): - """Get a text summmary of a numeric tensor. + """Get a text summary of a numeric tensor. This summary is only available for numeric (int*, float*, complex*) and Boolean tensors. diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py index 64b014a6b5..1131995b3e 100644 --- a/tensorflow/python/estimator/training.py +++ b/tensorflow/python/estimator/training.py @@ -199,7 +199,7 @@ class EvalSpec( evaluations on different data sets. Metrics for different evaluations are saved in separate folders, and appear separately in tensorboard. hooks: Iterable of `tf.train.SessionRunHook` objects to run - on all workers (including chief) during training. + during evaluation. exporters: Iterable of `Exporter`s, or a single one, or `None`. `exporters` will be invoked after each evaluation. start_delay_secs: Int. Start evaluating after waiting for this many @@ -408,8 +408,8 @@ def train_and_evaluate(estimator, train_spec, eval_spec): Args: estimator: An `Estimator` instance to train and evaluate. - train_spec: A `TrainSpec instance to specify the training specification. - eval_spec: A `EvalSpec instance to specify the evaluation and export + train_spec: A `TrainSpec` instance to specify the training specification. + eval_spec: A `EvalSpec` instance to specify the evaluation and export specification. Raises: diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index b02bae95fd..d8ecabcdea 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -903,6 +903,21 @@ cuda_py_test( ], ) +cuda_py_test( + name = "nth_element_op_test", + size = "small", + srcs = ["nth_element_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:gradients", + "//tensorflow/python:nn_grad", + "//tensorflow/python:nn_ops", + ], +) + tf_py_test( name = "unique_op_test", size = "small", diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py index 8ec93119f2..0c802476a0 100644 --- a/tensorflow/python/kernel_tests/batchtospace_op_test.py +++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py @@ -24,6 +24,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -52,14 +53,15 @@ class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl): def testDepthToSpaceTranspose(self): x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7]) block_size = 2 - crops = np.zeros((2, 2), dtype=np.int32) - y1 = self.batch_to_space(x, crops, block_size=block_size) - y2 = array_ops.transpose( - array_ops.depth_to_space( - array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size), - [3, 1, 2, 0]) - with self.test_session(): - self.assertAllEqual(y1.eval(), y2.eval()) + for crops_dtype in [dtypes.int64, dtypes.int32]: + crops = array_ops.zeros((2, 2), dtype=crops_dtype) + y1 = self.batch_to_space(x, crops, block_size=block_size) + y2 = array_ops.transpose( + array_ops.depth_to_space( + array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size), + [3, 1, 2, 0]) + with self.test_session(): + self.assertAllEqual(y1.eval(), y2.eval()) class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl): @@ -287,9 +289,10 @@ class BatchToSpaceGradientCppTest(BatchToSpaceGradientTest, CppOpImpl): class BatchToSpaceNDGradientTest(test.TestCase): # Check the gradients. - def _checkGrad(self, x, block_shape, crops): + def _checkGrad(self, x, block_shape, crops, crops_dtype): block_shape = np.array(block_shape) - crops = np.array(crops).reshape((len(block_shape), 2)) + crops = constant_op.constant( + np.array(crops).reshape((len(block_shape), 2)), crops_dtype) with self.test_session(): tf_x = ops.convert_to_tensor(x) tf_y = array_ops.batch_to_space_nd(tf_x, block_shape, crops) @@ -304,23 +307,26 @@ class BatchToSpaceNDGradientTest(test.TestCase): self.assertAllClose(x_jacob_t, x_jacob_n, rtol=1e-2, atol=epsilon) - def _compare(self, input_shape, block_shape, crops): + def _compare(self, input_shape, block_shape, crops, crops_dtype): input_shape = list(input_shape) input_shape[0] *= np.prod(block_shape) x = np.random.normal( 0, 1, np.prod(input_shape)).astype(np.float32).reshape(input_shape) - self._checkGrad(x, block_shape, crops) + self._checkGrad(x, block_shape, crops, crops_dtype) # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. def testSmall(self): - self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype) def testSmall2(self): - self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype) def testSmallCrop1x1(self): - self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py index f0b7885732..6cfa9b37fe 100644 --- a/tensorflow/python/kernel_tests/diag_op_test.py +++ b/tensorflow/python/kernel_tests/diag_op_test.py @@ -279,7 +279,7 @@ class MatrixDiagPartTest(test.TestCase): class DiagTest(test.TestCase): - def diagOp(self, diag, dtype, expected_ans, use_gpu=False): + def _diagOp(self, diag, dtype, expected_ans, use_gpu): with self.test_session(use_gpu=use_gpu): tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype))) out = tf_ans.eval() @@ -290,6 +290,10 @@ class DiagTest(test.TestCase): self.assertShapeEqual(expected_ans, tf_ans) self.assertShapeEqual(diag, tf_ans_inv) + def diagOp(self, diag, dtype, expected_ans): + self._diagOp(diag, dtype, expected_ans, False) + self._diagOp(diag, dtype, expected_ans, True) + def testEmptyTensor(self): x = np.array([]) expected_ans = np.empty([0, 0]) @@ -400,13 +404,53 @@ class DiagTest(test.TestCase): dtype=dtype) self.diagOp(x, dtype, expected_ans) + def testRankFourNumberTensor(self): + for dtype in [np.float32, np.float64, np.int64, np.int32]: + # Input with shape [2, 1, 2, 3] + x = np.array([[[[ 1, 2, 3], + [ 4, 5, 6]]], + [[[ 7, 8, 9], + [10, 11, 12]]]], dtype=dtype) + # Output with shape [2, 1, 2, 3, 2, 1, 2, 3] + expected_ans = np.array( + [[[[[[[[1, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 2, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 3], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]]], + [[[[[0, 0, 0], [4, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 5, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 6]]], + [[[0, 0, 0], [0, 0, 0]]]]]]], + + [[[[[[[0, 0, 0], [0, 0, 0]]], + [[[7, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 8, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 9], [0, 0, 0]]]]], + [[[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [10, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 11, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 12]]]]]]]], dtype=dtype) + self.diagOp(x, dtype, expected_ans) + + def testInvalidRank(self): + with self.assertRaisesRegexp(ValueError, "must be at least rank 1"): + array_ops.diag(0.0) + class DiagPartOpTest(test.TestCase): def setUp(self): np.random.seed(0) - def diagPartOp(self, tensor, dtype, expected_ans, use_gpu=False): + def _diagPartOp(self, tensor, dtype, expected_ans, use_gpu): with self.test_session(use_gpu=use_gpu): tensor = ops.convert_to_tensor(tensor.astype(dtype)) tf_ans_inv = array_ops.diag_part(tensor) @@ -414,6 +458,10 @@ class DiagPartOpTest(test.TestCase): self.assertAllClose(inv_out, expected_ans) self.assertShapeEqual(expected_ans, tf_ans_inv) + def diagPartOp(self, tensor, dtype, expected_ans): + self._diagPartOp(tensor, dtype, expected_ans, False) + self._diagPartOp(tensor, dtype, expected_ans, True) + def testRankTwoFloatTensor(self): x = np.random.rand(3, 3) i = np.arange(3) @@ -451,11 +499,23 @@ class DiagPartOpTest(test.TestCase): self.diagPartOp(x, np.float32, expected_ans) self.diagPartOp(x, np.float64, expected_ans) + def testRankEightComplexTensor(self): + x = np.random.rand(2, 2, 2, 3, 2, 2, 2, 3) + i = np.arange(2)[:, None, None, None] + j = np.arange(2)[:, None, None] + k = np.arange(2)[:, None] + l = np.arange(3) + expected_ans = x[i, j, k, l, i, j, k, l] + self.diagPartOp(x, np.complex64, expected_ans) + self.diagPartOp(x, np.complex128, expected_ans) + def testOddRank(self): w = np.random.rand(2) x = np.random.rand(2, 2, 2) self.assertRaises(ValueError, self.diagPartOp, w, np.float32, 0) self.assertRaises(ValueError, self.diagPartOp, x, np.float32, 0) + with self.assertRaises(ValueError): + array_ops.diag_part(0.0) def testUnevenDimensions(self): w = np.random.rand(2, 5) diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py index 4f053d2a21..ee86cf0b24 100644 --- a/tensorflow/python/kernel_tests/listdiff_op_test.py +++ b/tensorflow/python/kernel_tests/listdiff_op_test.py @@ -41,15 +41,17 @@ class ListDiffTest(test.TestCase): y = [compat.as_bytes(str(a)) for a in y] out = [compat.as_bytes(str(a)) for a in out] for diff_func in [array_ops.setdiff1d]: - with self.test_session() as sess: - x_tensor = ops.convert_to_tensor(x, dtype=dtype) - y_tensor = ops.convert_to_tensor(y, dtype=dtype) - out_tensor, idx_tensor = diff_func(x_tensor, y_tensor) - tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) - self.assertAllEqual(tf_out, out) - self.assertAllEqual(tf_idx, idx) - self.assertEqual(1, out_tensor.get_shape().ndims) - self.assertEqual(1, idx_tensor.get_shape().ndims) + for index_dtype in [dtypes.int32, dtypes.int64]: + with self.test_session() as sess: + x_tensor = ops.convert_to_tensor(x, dtype=dtype) + y_tensor = ops.convert_to_tensor(y, dtype=dtype) + out_tensor, idx_tensor = diff_func(x_tensor, y_tensor, + index_dtype=index_dtype) + tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) + self.assertAllEqual(tf_out, out) + self.assertAllEqual(tf_idx, idx) + self.assertEqual(1, out_tensor.get_shape().ndims) + self.assertEqual(1, idx_tensor.get_shape().ndims) def testBasic1(self): x = [1, 2, 3, 4] diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py index f21b0dfeab..e5b7cbce7a 100644 --- a/tensorflow/python/kernel_tests/metrics_test.py +++ b/tensorflow/python/kernel_tests/metrics_test.py @@ -3426,7 +3426,7 @@ class MeanIOUTest(test.TestCase): sess.run(variables.local_variables_initializer()) for _ in range(5): sess.run(update_op) - desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0, 0.]) + desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0]) self.assertAlmostEqual(desired_output, miou.eval()) def testUpdateOpEvalIsAccumulatedConfusionMatrix(self): @@ -3505,6 +3505,55 @@ class MeanIOUTest(test.TestCase): desired_miou = np.mean([2. / 4., 4. / 6.]) self.assertAlmostEqual(desired_miou, miou.eval()) + def testMissingClassInLabels(self): + labels = constant_op.constant([ + [[0, 0, 1, 1, 0, 0], + [1, 0, 0, 0, 0, 1]], + [[1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0]]]) + predictions = constant_op.constant([ + [[0, 0, 2, 1, 1, 0], + [0, 1, 2, 2, 0, 1]], + [[0, 0, 2, 1, 1, 1], + [1, 1, 2, 0, 0, 0]]]) + num_classes = 3 + with self.test_session() as sess: + miou, update_op = metrics.mean_iou(labels, predictions, num_classes) + sess.run(variables.local_variables_initializer()) + self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval()) + self.assertAlmostEqual( + 1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)), + miou.eval()) + + def testMissingClassOverallSmall(self): + labels = constant_op.constant([0]) + predictions = constant_op.constant([0]) + num_classes = 2 + with self.test_session() as sess: + miou, update_op = metrics.mean_iou(labels, predictions, num_classes) + sess.run(variables.local_variables_initializer()) + self.assertAllEqual([[1, 0], [0, 0]], update_op.eval()) + self.assertAlmostEqual(1, miou.eval()) + + def testMissingClassOverallLarge(self): + labels = constant_op.constant([ + [[0, 0, 1, 1, 0, 0], + [1, 0, 0, 0, 0, 1]], + [[1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0]]]) + predictions = constant_op.constant([ + [[0, 0, 1, 1, 0, 0], + [1, 1, 0, 0, 1, 1]], + [[0, 0, 0, 1, 1, 1], + [1, 1, 1, 0, 0, 0]]]) + num_classes = 3 + with self.test_session() as sess: + miou, update_op = metrics.mean_iou(labels, predictions, num_classes) + sess.run(variables.local_variables_initializer()) + self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval()) + self.assertAlmostEqual( + 1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval()) + class MeanPerClassAccuracyTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py new file mode 100644 index 0000000000..58cd46d2d5 --- /dev/null +++ b/tensorflow/python/kernel_tests/nth_element_op_test.py @@ -0,0 +1,174 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +import tensorflow.python.ops.nn_grad # pylint: disable=unused-import +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradients_impl +from tensorflow.python.platform import test + + +class NthElementTest(test.TestCase): + + def _validateNthElement(self, inputs, dtype, n, reverse, expected_values): + np_expected_values = np.array(expected_values) + with self.test_session(use_gpu=False) as sess: + inputs_op = ops.convert_to_tensor(inputs, dtype=dtype) + values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse) + values = sess.run(values_op) + + self.assertShapeEqual(np_expected_values, values_op) + self.assertAllClose(np_expected_values, values) + + def testExample1(self): + inputs = [2.2, 4.4, 1.1, 5.5, 3.3] + self._validateNthElement(inputs, dtypes.float32, 1, False, 2.2) + self._validateNthElement(inputs, dtypes.float32, 1, True, 4.4) + + def testExample2(self): + inputs = [[2.2, 4.4, 1.1], [5.5, 3.3, 6.6]] + self._validateNthElement(inputs, dtypes.float64, 2, False, [4.4, 6.6]) + self._validateNthElement(inputs, dtypes.float64, 2, True, [1.1, 3.3]) + + def testExample3(self): + inputs = [[[2, 4, 1], [5, -3, 6]], + [[7, 9, -8], [9, 0, 4]]] + self._validateNthElement(inputs, dtypes.int32, 0, False, + [[1, -3], [-8, 0]]) + self._validateNthElement(inputs, dtypes.int64, 0, True, + [[4, 6], [9, 9]]) + + def _testFloatLargeInput(self, input_shape): + inputs = np.random.random_sample(input_shape) + n = np.random.randint(input_shape[-1]) + sort_inputs = np.sort(inputs) + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.float32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.float64, n, True, expected_values) + + def _testIntLargeInput(self, input_shape): + inputs = np.random.randint(-1e3, 1e3, input_shape) + n = np.random.randint(input_shape[-1]) + sort_inputs = np.sort(inputs) + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.int32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.int64, n, True, expected_values) + + def _testLargeInput(self, input_shape): + self._testFloatLargeInput(input_shape) + self._testIntLargeInput(input_shape) + + def testLargeInput(self): + self._testLargeInput([1]) + self._testLargeInput([10]) + self._testLargeInput([5, 10]) + self._testLargeInput([50, 100]) + self._testLargeInput([50, 10000]) + self._testLargeInput([50, 10, 100]) + self._testLargeInput([50, 10, 10, 100]) + + def _testEnumerateN(self, input_shape): + inputs = np.random.random_sample(input_shape) + sort_inputs = np.sort(inputs) + for n in range(input_shape[-1]): + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.float32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.float64, n, True, expected_values) + + def testEnumerateN(self): + self._testEnumerateN([1]) + self._testEnumerateN([10]) + self._testEnumerateN([10, 10]) + self._testEnumerateN([10, 10, 10]) + self._testEnumerateN([10, 10, 10, 10]) + + def testInvalidInput(self): + with self.assertRaisesRegexp(ValueError, + "at least rank 1 but is rank 0"): + nn_ops.nth_element(5, 0) + + def testInvalidInputAtEval(self): + with self.test_session(use_gpu=False): + v = array_ops.placeholder(dtype=dtypes.float32) + with self.assertRaisesOpError("Input must be >= 1-D"): + nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0}) + + def testInvalidN(self): + with self.assertRaisesRegexp(ValueError, + "non-negative but is -1"): + nn_ops.nth_element([5], -1) + with self.assertRaisesRegexp(ValueError, + "scalar but has rank 1"): + nn_ops.nth_element([5, 6, 3], [1]) + + def testInvalidNAtEval(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.test_session(use_gpu=False): + n = array_ops.placeholder(dtypes.int32) + values = nn_ops.nth_element(inputs, n) + with self.assertRaisesOpError("Need n >= 0, got -7"): + values.eval(feed_dict={n: -7}) + + def testNTooLarge(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.assertRaisesRegexp(ValueError, + "must have last dimension > n = 2"): + nn_ops.nth_element(inputs, 2) + + def testNTooLargeAtEval(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.test_session(use_gpu=False): + n = array_ops.placeholder(dtypes.int32) + values = nn_ops.nth_element(inputs, n) + with self.assertRaisesOpError(r"Input must have at least n\+1 columns"): + values.eval(feed_dict={n: 2}) + + def testGradients(self): + with self.test_session(use_gpu=False) as sess: + inputs = array_ops.placeholder(dtypes.int32, shape=[3, 5]) + values = nn_ops.nth_element(inputs, 3) + grad = sess.run( + gradients_impl.gradients( + values, inputs, grad_ys=[[-1., 2., 5.]]), + feed_dict={inputs: [[2, -1, 1000, 3, 1000], + [1, 5, 2, 4, 3], + [2, 2, 2, 2, 2], + ]}) + self.assertAllClose(grad[0], [[0, 0, -0.5, 0, -0.5], + [0, 0, 0, 2, 0], + [1, 1, 1, 1, 1], + ]) + + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py index ca1f3f878f..2c766e3640 100644 --- a/tensorflow/python/kernel_tests/pad_op_test.py +++ b/tensorflow/python/kernel_tests/pad_op_test.py @@ -193,6 +193,25 @@ class PadOpTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Unknown padding mode"): array_ops.pad(x, [[1, 0], [2, 1]], mode="weird").eval() + def testPaddingTypes(self): + paddings = [[1, 0], [2, 3], [0, 2]] + inputs = np.random.randint(-100, 100, (4, 4, 3)).astype(np.float32) + for mode in ("CONSTANT", "REFLECT", "SYMMETRIC", "reflect", "symmetric", + "constant"): + for padding_dtype in [dtypes.int32, dtypes.int64]: + np_val = self._npPad(inputs, + paddings, + mode=mode, + constant_values=0) + with self.test_session(use_gpu=True): + tf_val = array_ops.pad(inputs, + constant_op.constant(paddings, padding_dtype), + mode=mode, + constant_values=0) + out = tf_val.eval() + self.assertAllEqual(np_val, out) + self.assertShapeEqual(np_val, tf_val) + def testIntTypes(self): # TODO(touts): Figure out why the padding tests do not work on GPU # for int types and rank > 2. @@ -284,6 +303,15 @@ class PadOpTest(test.TestCase): self.assertAllEqual(inp, out) self.assertShapeEqual(inp, tf_val) + def testPadTypes(self): + for dtype in [dtypes.int32, dtypes.int64]: + paddings = np.zeros((0, 2)) + inp = np.asarray(7) + with self.test_session(use_gpu=True): + tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype)) + out = tf_val.eval() + self.assertAllEqual(inp, out) + self.assertShapeEqual(inp, tf_val) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index c794351fe9..2dc65b1384 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -163,6 +163,13 @@ class SumReductionTest(BaseReductionTest): reduction_axes = tuple(reduction_axes) return np.sum(x, axis=reduction_axes, keepdims=keep_dims) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -193,6 +200,7 @@ class SumReductionTest(BaseReductionTest): tf_out_mean = sess.run(tf_mean) self.assertAllClose(tf_out_mean, 1.) + def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) @@ -369,6 +377,13 @@ class MeanReductionTest(BaseReductionTest): return np_sum // count return np_sum / count + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -435,6 +450,13 @@ class ProdReductionTest(BaseReductionTest): reduction_axes = tuple(reduction_axes) return np.prod(x, axis=reduction_axes, keepdims=keep_dims) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -531,6 +553,13 @@ class MinReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -637,6 +666,13 @@ class MaxReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -757,6 +793,14 @@ class AllReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_all([True, True], + constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, True) + def testAll3D(self): # Create a 3D array of bools and reduce across all possible # dimensions @@ -798,6 +842,14 @@ class AnyReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_any([True, True], + constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, True) + def testAll3D(self): # Create a 3D array of bools and reduce across all possible # dimensions diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py index 6b2b589a06..08b4a2aaae 100644 --- a/tensorflow/python/kernel_tests/scan_ops_test.py +++ b/tensorflow/python/kernel_tests/scan_ops_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import gradient_checker @@ -92,6 +94,14 @@ class CumsumTest(test.TestCase): for axis in (-1, 0): self._compareAll(x, axis) + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True): + axis = constant_op.constant(0, axis_dtype) + tf_out = math_ops.cumsum(x, axis).eval() + def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) @@ -190,6 +200,14 @@ class CumprodTest(test.TestCase): for axis in (-1, 0): self._compareAll(x, axis) + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True): + axis = constant_op.constant(0, axis_dtype) + tf_out = math_ops.cumprod(x, axis).eval() + def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py index 52cf904528..a9fc699b21 100644 --- a/tensorflow/python/kernel_tests/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/shape_ops_test.py @@ -411,14 +411,16 @@ class TileTest(test.TestCase): self.assertEqual(7, result) def testSimple(self): - with self.test_session(): - inp = np.random.rand(4, 1).astype(np.float32) - a = constant_op.constant(inp) - tiled = array_ops.tile(a, [1, 4]) - result = tiled.eval() - self.assertEqual(result.shape, (4, 4)) - self.assertEqual([4, 4], tiled.get_shape()) - self.assertTrue((result == np.tile(inp, (1, 4))).all()) + # multiples could be int32 or int64 + for dtype in [dtypes.int32, dtypes.int64]: + with self.test_session(use_gpu=True): + inp = np.random.rand(4, 1).astype(np.float32) + a = constant_op.constant(inp) + tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype)) + result = tiled.eval() + self.assertEqual(result.shape, (4, 4)) + self.assertEqual([4, 4], tiled.get_shape()) + self.assertTrue((result == np.tile(inp, (1, 4))).all()) def testIdentityTileAndGrad(self): with self.test_session(): diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py index f6997e9c61..f415d9e70d 100644 --- a/tensorflow/python/kernel_tests/slice_op_test.py +++ b/tensorflow/python/kernel_tests/slice_op_test.py @@ -217,6 +217,17 @@ class SliceTest(test.TestCase): self.assertEqual(expected_val.shape, slice_t.get_shape()) self.assertEqual(expected_val.shape, slice2_t.get_shape()) + def testPartialShapeInference(self): + z = array_ops.zeros((1, 2, 3)) + self.assertAllEqual(z.get_shape().as_list(), [1, 2, 3]) + + m1 = array_ops.slice(z, [0, 0, 0], [-1, -1, -1]) + self.assertAllEqual(m1.get_shape().as_list(), [1, 2, 3]) + + m2 = array_ops.slice(z, [0, 0, 0], [constant_op.constant(1) + 0, 2, -1]) + self.assertAllEqual(m2.get_shape().as_list(), [None, 2, None]) + + def _testGradientSlice(self, input_shape, slice_begin, slice_size): with self.test_session(use_gpu=True): num_inputs = np.prod(input_shape) diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py index 3b352937c8..c551d9c3d0 100644 --- a/tensorflow/python/kernel_tests/transpose_op_test.py +++ b/tensorflow/python/kernel_tests/transpose_op_test.py @@ -317,6 +317,19 @@ class TransposeTest(test.TestCase): np.arange(0, 8).reshape([2, 4]).astype(np.float32), np.array([1, 0]).astype(np.int32)) + def testPermType(self): + for perm_dtype in [np.int64, np.int32]: + x = np.arange(0, 8).reshape([2, 4]).astype(np.float32) + p = np.array([1, 0]).astype(perm_dtype) + np_ans = np.copy(x).transpose(p) + with self.test_session(use_gpu=True): + inx = ops.convert_to_tensor(x) + inp = constant_op.constant(p) + y = array_ops.transpose(inx, inp) + tf_ans = y.eval() + self.assertShapeEqual(np_ans, y) + self.assertAllEqual(np_ans, tf_ans) + def testHalf(self): self._compare(np.arange(0, 21).reshape([3, 7]).astype(np.float16)) self._compare(np.arange(0, 210).reshape([2, 3, 5, 7]).astype(np.float16)) diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt index fcd378e3c0..86bc038e86 100644 --- a/tensorflow/python/ops/hidden_ops.txt +++ b/tensorflow/python/ops/hidden_ops.txt @@ -43,6 +43,7 @@ UniformCandidateSampler GenerateVocabRemapping LoadAndRemapMatrix + # control_flow_ops Switch Merge @@ -241,6 +242,7 @@ TensorSummaryV2 # math_ops Abs +AccumulateNV2 AddN All Any @@ -257,6 +259,7 @@ ComplexAbs Conj FloorDiv FloorMod +HistogramFixedWidth Max Mean Min diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py index c2077d51af..51e4be9343 100644 --- a/tensorflow/python/ops/histogram_ops.py +++ b/tensorflow/python/ops/histogram_ops.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops @@ -69,30 +70,6 @@ def histogram_fixed_width(values, ``` """ with ops.name_scope(name, 'histogram_fixed_width', - [values, value_range, nbins]) as scope: - values = ops.convert_to_tensor(values, name='values') - values = array_ops.reshape(values, [-1]) - value_range = ops.convert_to_tensor(value_range, name='value_range') - nbins = ops.convert_to_tensor(nbins, dtype=dtypes.int32, name='nbins') - nbins_float = math_ops.cast(nbins, values.dtype) - - # Map tensor values that fall within value_range to [0, 1]. - scaled_values = math_ops.truediv(values - value_range[0], - value_range[1] - value_range[0], - name='scaled_values') - - # map tensor values within the open interval value_range to {0,.., nbins-1}, - # values outside the open interval will be zero or less, or nbins or more. - indices = math_ops.floor(nbins_float * scaled_values, name='indices') - - # Clip edge cases (e.g. value = value_range[1]) or "outliers." - indices = math_ops.cast( - clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32) - - # TODO(langmore) This creates an array of ones to add up and place in the - # bins. This is inefficient, so replace when a better Op is available. - return math_ops.unsorted_segment_sum( - array_ops.ones_like(indices, dtype=dtype), - indices, - nbins, - name=scope) + [values, value_range, nbins]) as name: + return gen_math_ops._histogram_fixed_width(values, value_range, nbins, + dtype=dtype, name=name) diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index e819e0234d..bf6e0296f6 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -36,7 +36,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [] expected_bin_counts = [0, 0, 0, 0, 0] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) @@ -47,7 +47,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width( values, value_range, nbins=5, dtype=dtypes.int64) self.assertEqual(dtypes.int64, hist.dtype) @@ -59,7 +59,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = np.float64([0.0, 5.0]) values = np.float64([-1.0, 0.0, 1.5, 2.0, 5.0, 15]) expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) @@ -70,7 +70,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]] expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index ebbf581204..d1554b399f 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -1374,6 +1374,25 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): y = image_ops.pad_to_bounding_box(image, 0, 0, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) + def testInt64(self): + x = [1, 2, 3, + 4, 5, 6, + 7, 8, 9] + x_shape = [3, 3, 1] + + y = [0, 0, 0, + 1, 2, 3, + 4, 5, 6, + 7, 8, 9] + y_shape = [4, 3, 1] + x = np.array(x).reshape(x_shape) + y = np.array(y).reshape(y_shape) + + i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64) + y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3]) + with self.test_session(use_gpu=True): + self.assertAllClose(y, y_tf.eval()) + def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) @@ -1672,8 +1691,8 @@ class ResizeImagesTest(test_util.TensorFlowTestCase): image_ops.ResizeMethod.BICUBIC, image_ops.ResizeMethod.AREA] - TYPES = [np.uint8, np.int8, np.int16, np.int32, np.int64, - np.float32, np.float64] + TYPES = [np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64, + np.float16, np.float32, np.float64] def _assertShapeInference(self, pre_shape, size, post_shape): # Try single image resize @@ -2434,9 +2453,13 @@ class JpegTest(test_util.TensorFlowTestCase): y, x, h, w = crop_window image1_crop = image_ops.crop_to_bounding_box(image1, y, x, h, w) - # Combined crop+decode. + # Combined decode+crop. image2 = image_ops.decode_and_crop_jpeg(jpeg0, crop_window) + # Combined decode+crop should have the same shape inference + self.assertAllEqual(image1_crop.get_shape().as_list(), + image2.get_shape().as_list()) + # CropAndDecode should be equal to DecodeJpeg+Crop. image1_crop, image2 = sess.run([image1_crop, image2]) self.assertAllEqual(image1_crop, image2) diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py index 752d260fba..55a18d28ca 100644 --- a/tensorflow/python/ops/losses/losses_impl.py +++ b/tensorflow/python/ops/losses/losses_impl.py @@ -27,6 +27,7 @@ from tensorflow.python.ops import nn from tensorflow.python.ops import nn_ops from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.ops.losses import util +from tensorflow.python.util.deprecation import deprecated_args class Reduction(object): @@ -230,10 +231,12 @@ def absolute_difference( losses, weights, scope, loss_collection, reduction=reduction) +@deprecated_args(None, "dim is deprecated, use axis instead", "dim") def cosine_distance( - labels, predictions, dim=None, weights=1.0, scope=None, + labels, predictions, axis=None, weights=1.0, scope=None, loss_collection=ops.GraphKeys.LOSSES, - reduction=Reduction.SUM_BY_NONZERO_WEIGHTS): + reduction=Reduction.SUM_BY_NONZERO_WEIGHTS, + dim=None): """Adds a cosine-distance loss to the training procedure. Note that the function assumes that `predictions` and `labels` are already @@ -242,13 +245,14 @@ def cosine_distance( Args: labels: `Tensor` whose shape matches 'predictions' predictions: An arbitrary matrix. - dim: The dimension along which the cosine distance is computed. + axis: The dimension along which the cosine distance is computed. weights: Optional `Tensor` whose rank is either 0, or the same rank as `labels`, and must be broadcastable to `labels` (i.e., all dimensions must be either `1`, or the same as the corresponding `losses` dimension). scope: The scope for the operations performed in computing the loss. loss_collection: collection to which this loss will be added. reduction: Type of reduction to apply to loss. + dim: The old (deprecated) name for `axis`. Returns: Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same @@ -256,10 +260,14 @@ def cosine_distance( Raises: ValueError: If `predictions` shape doesn't match `labels` shape, or - `dim`, `labels`, `predictions` or `weights` is `None`. + `axis`, `labels`, `predictions` or `weights` is `None`. """ - if dim is None: - raise ValueError("`dim` cannot be None.") + if dim is not None: + if axis is not None: + raise ValueError("Cannot specify both 'axis' and 'dim'") + axis = dim + if axis is None and dim is None: + raise ValueError("You must specify 'axis'.") if labels is None: raise ValueError("labels must not be None.") if predictions is None: @@ -271,7 +279,7 @@ def cosine_distance( predictions.get_shape().assert_is_compatible_with(labels.get_shape()) radial_diffs = math_ops.multiply(predictions, labels) - losses = 1 - math_ops.reduce_sum(radial_diffs, axis=(dim,), keep_dims=True) + losses = 1 - math_ops.reduce_sum(radial_diffs, axis=(axis,), keep_dims=True) return compute_weighted_loss( losses, weights, scope, loss_collection, reduction=reduction) diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py index 9273659a77..10ff4be2dd 100644 --- a/tensorflow/python/ops/metrics_impl.py +++ b/tensorflow/python/ops/metrics_impl.py @@ -949,6 +949,12 @@ def mean_iou(labels, cm_diag = math_ops.to_float(array_ops.diag_part(total_cm)) denominator = sum_over_row + sum_over_col - cm_diag + # The mean is only computed over classes that appear in the + # label or prediction tensor. If the denominator is 0, we need to + # ignore the class. + num_valid_entries = math_ops.reduce_sum(math_ops.cast( + math_ops.not_equal(denominator, 0), dtype=dtypes.float32)) + # If the value of the denominator is 0, set it to 1 to avoid # zero division. denominator = array_ops.where( @@ -956,7 +962,13 @@ def mean_iou(labels, denominator, array_ops.ones_like(denominator)) iou = math_ops.div(cm_diag, denominator) - return math_ops.reduce_mean(iou, name=name) + + # If the number of valid entries is 0 (no classes) we return 0. + result = array_ops.where( + math_ops.greater(num_valid_entries, 0), + math_ops.reduce_sum(iou, name=name) / num_valid_entries, + 0) + return result mean_iou_v = compute_mean_iou('mean_iou') diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index af610d8fdb..557f39fb42 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -352,6 +352,13 @@ def _Relu6Grad(op, grad): return gen_nn_ops._relu6_grad(grad, op.outputs[0]) # pylint: disable=protected-access +@ops.RegisterGradient("Relu6Grad") +def _Relu6GradGrad(op, grad): + x = op.inputs[1] + return (gen_nn_ops._relu6_grad(grad, x), array_ops.zeros( + shape=array_ops.shape(x), dtype=x.dtype)) + + @ops.RegisterGradient("Elu") def _EluGrad(op, grad): return gen_nn_ops._elu_grad(grad, op.outputs[0]) @@ -934,3 +941,32 @@ def _TopKGrad(op, grad, _): validate_indices=False), in_shape), array_ops.zeros( [], dtype=dtypes.int32)] + + +@ops.RegisterGradient("NthElement") +def _NthElementGrad(op, grad): + """Return the gradients for NthElement. + + Args: + op: The NthElementOp for which we need to generate gradients. + grad: Tensor. The gradients passed to the NthElementOp + + Returns: + A list of two tensors, the first being the gradient w.r.t. the input, + the second being the gradient w.r.t. the N (None). + """ + input = op.inputs[0] + output = op.outputs[0] + + # Compute the number of elements which equal to output in each reduction + # dimension. If there are multiple elements then the gradient will be + # divided between them. + indicators = math_ops.cast( + math_ops.equal(array_ops.expand_dims(output, -1), input), + grad.dtype) + + grad = array_ops.expand_dims(grad, -1) + num_selected = array_ops.expand_dims( + math_ops.reduce_sum(indicators, -1), -1) + + return [math_ops.div(indicators, num_selected) * grad, None] diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py new file mode 100644 index 0000000000..f7541c0e89 --- /dev/null +++ b/tensorflow/python/ops/nn_grad_test.py @@ -0,0 +1,48 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Python ops defined in nn_grad.py.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import nn_grad +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import test + + +class Relu6OpTest(test.TestCase): + def testRelu6GradGrad(self): + inputs = constant_op.constant([[-2, -1, 1, 3], [5, 7, 8, 9]], + dtype=dtypes.float32) + x_init_value = np.array([[-3.5, -1.5, 2, 4], [4.5, 7.5, 8.5, 11]]) + r = nn_ops.relu6(inputs) + r_g = gradients_impl.gradients(r, inputs)[0] + with self.test_session(): + error = gradient_checker.compute_gradient_error( + inputs, inputs.get_shape().as_list(), + r_g, r_g.get_shape().as_list(), + x_init_value=x_init_value) + self.assertLess(error, 1e-4) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 5f82323bfc..a37b68c6fa 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -2140,6 +2140,34 @@ def top_k(input, k=1, sorted=True, name=None): return gen_nn_ops._top_kv2(input, k=k, sorted=sorted, name=name) +def nth_element(input, n, reverse=False, name=None): + r"""Finds values of the `n`-th order statistic for the last dmension. + + If the input is a vector (rank-1), finds the entries which is the nth-smallest + value in the vector and outputs their values as scalar tensor. + + For matrices (resp. higher rank input), computes the entries which is the + nth-smallest value in each row (resp. vector along the last dimension). Thus, + + values.shape = input.shape[:-1] + + Args: + input: 1-D or higher `Tensor` with last dimension at least `n+1`. + n: A `Tensor` of type `int32`. + 0-D. Position of sorted vector to select along the last dimension (along + each row for matrices). Valid range of n is `[0, input.shape[:-1])` + reverse: An optional `bool`. Defaults to `False`. + When set to True, find the nth-largest value in the vector and vice + versa. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + The `n`-th order statistic along each last dimensional slice. + """ + return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name) + + def conv1d(value, filters, stride, padding, use_cudnn_on_gpu=None, data_format=None, name=None): diff --git a/tensorflow/python/platform/self_check.py b/tensorflow/python/platform/self_check.py index 39d38d7bbc..966a094e55 100644 --- a/tensorflow/python/platform/self_check.py +++ b/tensorflow/python/platform/self_check.py @@ -21,7 +21,13 @@ from __future__ import print_function import os -from tensorflow.python.platform import build_info +try: + from tensorflow.python.platform import build_info +except ImportError: + raise ImportError("Could not import tensorflow. Do not import tensorflow " + "from its source directory; change directory to outside " + "the TensorFlow source tree, and relaunch your Python " + "interpreter from there.") def preload_check(): diff --git a/tensorflow/tools/api/golden/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/tensorflow.losses.pbtxt index 79443839b9..c1d190ae11 100644 --- a/tensorflow/tools/api/golden/tensorflow.losses.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.losses.pbtxt @@ -18,7 +18,7 @@ tf_module { } member_method { name: "cosine_distance" - argspec: "args=[\'labels\', \'predictions\', \'dim\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], " + argspec: "args=[\'labels\', \'predictions\', \'axis\', \'weights\', \'scope\', \'loss_collection\', \'reduction\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\', \'None\'], " } member_method { name: "get_losses" diff --git a/tensorflow/tools/ci_build/Dockerfile.pi b/tensorflow/tools/ci_build/Dockerfile.pi index 9d12ededb8..2fddd6a2c0 100644 --- a/tensorflow/tools/ci_build/Dockerfile.pi +++ b/tensorflow/tools/ci_build/Dockerfile.pi @@ -14,6 +14,9 @@ RUN /install/install_proto3.sh RUN /install/install_buildifier.sh RUN /install/install_auditwheel.sh RUN /install/install_golang.sh + +# The following line installs the Python cross-compilation toolchain. All the +# preceding dependencies should be kept in sync with the main CPU docker file. RUN /install/install_pi_toolchain.sh # Set up the master bazelrc configuration file. diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3 new file mode 100644 index 0000000000..18b131ea19 --- /dev/null +++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3 @@ -0,0 +1,23 @@ +FROM ubuntu:14.04 + +MAINTAINER Jan Prach + +# Copy and run the install scripts. +COPY install/*.sh /install/ +RUN /install/install_bootstrap_deb_packages.sh +RUN add-apt-repository -y ppa:openjdk-r/ppa && \ + add-apt-repository -y ppa:george-edison55/cmake-3.x +RUN /install/install_deb_packages.sh +RUN /install/install_pip_packages.sh +RUN /install/install_bazel.sh +RUN /install/install_proto3.sh +RUN /install/install_buildifier.sh +RUN /install/install_auditwheel.sh +RUN /install/install_golang.sh + +# The following line installs the Python cross-compilation toolchain. All the +# preceding dependencies should be kept in sync with the main CPU docker file. +RUN /install/install_pi_python3_toolchain.sh + +# Set up the master bazelrc configuration file. +COPY install/.bazelrc /etc/bazel.bazelrc diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md index ad83669950..acef833909 100644 --- a/tensorflow/tools/ci_build/README.md +++ b/tensorflow/tools/ci_build/README.md @@ -1,115 +1,76 @@ # TensorFlow Builds -This directory contains all the files and setup instructions to run all -the important builds and tests. **You can trivially run it yourself!** It also -run continuous integration [ci.tensorflow.org](https://ci.tensorflow.org). - - +This directory contains all the files and setup instructions to run all the +important builds and tests. You can run it yourself! ## Run It Yourself -1. Install [Docker](http://www.docker.com/). Follow instructions - [on the Docker site](https://docs.docker.com/installation/). - - You can run all the jobs **without docker** if you are on mac or on linux - and you just don't want docker. Just install all the dependencies from - [Installing TensorFlow](https://www.tensorflow.org/install/). - Then run any of the one liners below without the - `tensorflow/tools/ci_build/ci_build.sh` in them. - -2. Clone tensorflow repository. - - ```bash - git clone https://github.com/tensorflow/tensorflow.git - ``` - -3. Go to tensorflow directory - - ```bash - cd tensorflow - ``` - -4. Build what you want, for example - - ```bash - tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... - ``` - If you are using the Docker image on Windows or OS X, the Docker VM's default - memory limit may be too low to build TensorFlow. This can result in - strange-looking errors, e.g. the compilation may fail with `gcc: internal - compiler error: Killed (program cc1plus)`. Try increasing the memory limit in - the Docker preferences. - - -## Jobs - -The jobs run by [ci.tensorflow.org](https://ci.tensorflow.org) include following: - -```bash -# Note: You can run the following one-liners yourself if you have Docker. Run -# without `tensorflow/tools/ci_build/ci_build.sh` on mac or linux without Docker. - -# build and run cpu tests -tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... +You have two options when running TensorFlow tests locally on your +machine. First, using docker, you can run our Continuous Integration +(CI) scripts on tensorflow devel images. The other option is to install +all TensorFlow dependencies on your machine and run the scripts +natively on your system. -# build and run gpu tests (note if you get unstable results you may be running -# out of gpu memory - if so add "--jobs=1" argument) -tensorflow/tools/ci_build/ci_build.sh GPU bazel test -c opt --config=cuda //tensorflow/... +### Run TensorFlow CI Scripts using Docker -# build pip with gpu support -tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda +1. Install Docker following the [instructions on the docker website](https://docs.docker.com/engine/installation/). -# build and run gpu tests using python 3 -CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda +2. Start a container with one of the devel images here: + https://hub.docker.com/r/tensorflow/tensorflow/tags/. -# build android example app -tensorflow/tools/ci_build/ci_build.sh ANDROID tensorflow/tools/ci_build/builds/android.sh +3. Based on your choice of the image, pick one of the scripts under + https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/ci_build/linux + and run them from the TensorFlow repository root. -# cmake cpu build and test -tensorflow/tools/ci_build/ci_build.sh CPU tensorflow/tools/ci_build/builds/cmake.sh +### Run TensorFlow CI Scripts Natively on your Machine -# run bash inside the container -CI_DOCKER_EXTRA_PARAMS='-it --rm' tensorflow/tools/ci_build/ci_build.sh CPU /bin/bash -``` +1. Follow the instructions at https://www.tensorflow.org/install/install_sources, + but stop when you get to the section "Configure the installation". You do not + need to configure the installation to run the CI scripts. -**Note**: The set of jobs and how they are triggered is still evolving. -There are builds for master branch on cpu, gpu and android. There is a build -for incoming gerrit changes. Gpu tests and benchmark are coming soon. Check -[ci.tensorflow.org](https://ci.tensorflow.org) for current jobs. +2. Pick the appropriate OS and python version you have installed, + and run the script under tensorflow/tools/ci_build/. +## TensorFlow Continuous Integration +To verify that new changes don’t break TensorFlow, we run builds and +tests on either [Jenkins](https://jenkins-ci.org/) or a CI system +internal to Google. -## How Does TensorFlow Continuous Integration Work +We can trigger builds and tests on updates to master or on each pull +request. Contact one of the repository maintainers to trigger builds +on your pull request. -We use [jenkins](https://jenkins-ci.org/) as our continuous integration. -It is running at [ci.tensorflow.org](https://ci.tensorflow.org). -All the jobs are run within [docker](http://www.docker.com/) containers. +### View CI Results -Builds can be triggered by push to master, push a change set or manually. -The build started in jenkins will first pull the git tree. Then jenkins builds -a docker container (using one of those Dockerfile.* files in this directory). -The build itself is run within the container itself. +The Pull Request will show if the change passed or failed the checks. -Source tree lives in jenkins job workspace. Docker container for jenkins -are transient - deleted after the build. Containers build very fast thanks -to docker caching. Individual builds are fast thanks to bazel caching. +From the pull request, click **Show all checks** to see the list of builds +and tests. Click on **Details** to see the results from Jenkins or the internal +CI system. +Results from Jenkins are displayed in the Jenkins UI. For more information, +see the [Jenkns documentation](https://jenkins.io/doc/). +Results from the internal CI system are displayed in the Build Status UI. In +this UI, to see the logs for a failed build: -## Implementation Details +* Click on the **INVOCATION LOG** tab to see the invocation log. -* The ci_build.sh script create and run docker container with all dependencies. - The builds/with_the_same_user together with ci_build.sh creates an environment - which is the same inside the container as it is outside. The same user, group, - path, so that docker symlinks work inside and outside the container. You can - use it for your development. Edit files in your git clone directory. If you - run the ci_build.sh it gets this directory mapped inside the container and - build your tree. +* Click on the **ARTIFACTS** tab to see a list of all artifacts, including logs. -* The unusual `bazel-ci_build-cache` directory is mapped to docker container - performing the build using docker's --volume parameter. This way we cache - bazel output between builds. +* Individual test logs may be available. To see these logs, from the **TARGETS** + tab, click on the failed target. Then, click on the **TARGET LOG** tab to see + its test log. -* The `builds` directory within this folder contains shell scripts to run within - the container. They essentially contains workarounds for current limitations - of bazel. + If you’re looking at target that is sharded or a test that is flaky, then + the build tool divided the target into multiple shards or ran the test + multiple times. Each test log is specific to the shard, run, and attempt. + To see a specific log: + + 1. Click on the log icon that is on the right next to the shard, run, + and attempt number. + + 2. In the grid that appears on the right, click on the specific shard, + run, and attempt to view its log. You can also type the desired shard, + run, or attempt number in the field above its grid. diff --git a/tensorflow/tools/ci_build/builds/android_full.sh b/tensorflow/tools/ci_build/builds/android_full.sh index 63250e0a4d..9d449241e8 100755 --- a/tensorflow/tools/ci_build/builds/android_full.sh +++ b/tensorflow/tools/ci_build/builds/android_full.sh @@ -40,7 +40,7 @@ rm -rf ${AAR_LIB_TMP} for CPU in ${CPUS//,/ } do echo "========== Building native libs for Android ${CPU} ==========" - bazel build -c opt --cpu=${CPU} \ + bazel build -c opt --config=monolithic --cpu=${CPU} \ --crosstool_top=//external:android/crosstool \ --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ //tensorflow/core:android_tensorflow_lib \ @@ -62,7 +62,7 @@ done # in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334) # TODO(gunan): remove extra flags once sandboxing is enabled for all builds. echo "========== Building TensorFlow Android Jar and Demo ==========" -bazel --bazelrc=/dev/null build -c opt --fat_apk_cpu=${CPUS} \ +bazel --bazelrc=/dev/null build -c opt --config=monolithic --fat_apk_cpu=${CPUS} \ --spawn_strategy=sandboxed --genrule_strategy=sandboxed \ //tensorflow/contrib/android:android_tensorflow_inference_java \ //tensorflow/contrib/android:android_tensorflow_inference_java.aar \ diff --git a/tensorflow/tools/ci_build/builds/libtensorflow.sh b/tensorflow/tools/ci_build/builds/libtensorflow.sh index 5052d3626c..26713dded8 100755 --- a/tensorflow/tools/ci_build/builds/libtensorflow.sh +++ b/tensorflow/tools/ci_build/builds/libtensorflow.sh @@ -78,9 +78,52 @@ function build_libtensorflow_tarball() { //tensorflow/tools/lib_package:libtensorflow_proto.zip mkdir -p ${DIR} + cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz ${DIR}/libtensorflow_jni${TARBALL_SUFFIX}.tar.gz - cp bazel-bin/tensorflow/java/libtensorflow.jar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR} + cp bazel-bin/tensorflow/java/libtensorflow.jar ${DIR} + cp_normalized_srcjar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}/libtensorflow-src.jar cp bazel-genfiles/tensorflow/tools/lib_package/libtensorflow_proto.zip ${DIR} chmod -x ${DIR}/* } + +# Helper function to copy a srcjar after moving any source files +# directly under the root to the "maven-style" src/main/java layout +# +# Source files generated by annotation processors appear directly +# under the root of srcjars jars created by bazel, rather than under +# the maven-style src/main/java subdirectory. +# +# Bazel manages annotation generated source as follows: First, it +# calls javac with options that create generated files under a +# bazel-out directory. Next, it archives the generated source files +# into a srcjar directly under the root. There doesn't appear to be a +# simple way to parameterize this from bazel, hence this helper to +# "normalize" the srcjar layout. +# +# Arguments: +# src_jar - path to the original srcjar +# dest_jar - path to the destination +# Returns: +# None +function cp_normalized_srcjar() { + local src_jar="$1" + local dest_jar="$2" + if [[ -z "${src_jar}" || -z "${dest_jar}" ]]; then + echo "Unexpected: missing arguments" >&2 + exit 2 + fi + local tmp_dir + tmp_dir=$(mktemp -d) + cp "${src_jar}" "${tmp_dir}/orig.jar" + pushd "${tmp_dir}" + # Extract any src/ files + jar -xf "${tmp_dir}/orig.jar" src/ + # Extract any org/ files under src/main/java + (mkdir -p src/main/java && cd src/main/java && jar -xf "${tmp_dir}/orig.jar" org/) + # Repackage src/ + jar -cMf "${tmp_dir}/new.jar" src + popd + cp "${tmp_dir}/new.jar" "${dest_jar}" + rm -rf "${tmp_dir}" +} diff --git a/tensorflow/tools/ci_build/install/install_golang.sh b/tensorflow/tools/ci_build/install/install_golang.sh index 596265b069..55c1674495 100755 --- a/tensorflow/tools/ci_build/install/install_golang.sh +++ b/tensorflow/tools/ci_build/install/install_golang.sh @@ -16,7 +16,7 @@ set -ex -GOLANG_URL="https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz" +GOLANG_URL="https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz" sudo mkdir -p /usr/local wget -q -O - "${GOLANG_URL}" | sudo tar -C /usr/local -xz diff --git a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh new file mode 100755 index 0000000000..9d8e3df3b5 --- /dev/null +++ b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +dpkg --add-architecture armhf +echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list +echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-updates main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list +echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-security main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list +echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list +sed -i 's#deb http://archive.ubuntu.com/ubuntu/#deb [arch=amd64] http://archive.ubuntu.com/ubuntu/#g' /etc/apt/sources.list +apt-get update +apt-get install -y libpython3-all-dev:armhf +echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list +curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - +apt-get update +rm -rf /usr/local/bin/bazel +apt-get install -y bazel python3 python3-numpy python3-dev python3-pip diff --git a/tensorflow/tools/ci_build/install/install_pi_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_toolchain.sh index ef30ba58c2..03c43cc838 100755 --- a/tensorflow/tools/ci_build/install/install_pi_toolchain.sh +++ b/tensorflow/tools/ci_build/install/install_pi_toolchain.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh new file mode 100755 index 0000000000..dbf376be6f --- /dev/null +++ b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +set -e +set -x + +N_JOBS=$(grep -c ^processor /proc/cpuinfo) + +echo "" +echo "Bazel will use ${N_JOBS} concurrent job(s)." +echo "" + +# Run configure. +export TF_NEED_CUDA=0 +export PYTHON_BIN_PATH=`which python2` +yes "" | $PYTHON_BIN_PATH configure.py + +# Run bazel test command. Double test timeouts to avoid flakes. +bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=py -k \ + --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only \ + --config=mkl --config=opt --test_output=errors -- \ + //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index 07a972400d..024cb40eb4 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:16.04 -MAINTAINER Craig Citro +LABEL maintainer="Craig Citro " # Pick up some TF dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index 60a94504b7..20e1dcd085 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -1,6 +1,6 @@ FROM ubuntu:16.04 -MAINTAINER Craig Citro +LABEL maintainer="Craig Citro " RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -72,7 +72,7 @@ RUN mkdir /bazel && \ RUN git clone https://github.com/tensorflow/tensorflow.git && \ cd tensorflow && \ - git checkout r1.3 + git checkout r1.4 WORKDIR /tensorflow # TODO(craigcitro): Don't install the pip package, since it makes it diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 04773376e9..21a44ee404 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -1,6 +1,6 @@ FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04 -MAINTAINER Craig Citro +LABEL maintainer="Craig Citro " RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -73,20 +73,23 @@ RUN mkdir /bazel && \ RUN git clone https://github.com/tensorflow/tensorflow.git && \ cd tensorflow && \ - git checkout r1.3 + git checkout r1.4 WORKDIR /tensorflow # Configure the build for our CUDA configuration. ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1 -RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 -RUN tensorflow/tools/ci_build/builds/configured GPU \ - bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ + LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ + tensorflow/tools/ci_build/builds/configured GPU \ + bazel build -c opt --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ tensorflow/tools/pip_package:build_pip_package && \ + rm /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \ pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \ rm -rf /tmp/pip && \ diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 index ac1a437031..4558bc5293 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 @@ -42,6 +42,7 @@ RUN pip --no-cache-dir install \ scipy \ sklearn \ pandas \ + wheel \ && \ python -m ipykernel.kernelspec @@ -80,22 +81,32 @@ RUN git clone https://github.com/tensorflow/tensorflow.git && \ WORKDIR /tensorflow # Configure the build for our CUDA configuration. -ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH -ENV TF_NEED_CUDA 1 -ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1 -ENV TF_CUDA_VERSION 9.0 -ENV TF_CUDNN_VERSION 7.0 +ENV CI_BUILD_PYTHON=python \ + LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:${LD_LIBRARY_PATH} \ + CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu \ + PYTHON_BIN_PATH=/usr/bin/python \ + PYTHON_LIB_PATH=/usr/local/lib/python2.7/dist-packages \ + TF_NEED_CUDA=1 \ + TF_CUDA_VERSION=9.0 \ + TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1,7.0 \ + TF_CUDNN_VERSION=7 RUN ./configure -RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ - bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ - --jobs=${TF_AVAILABLE_CPUS} \ - tensorflow/tools/pip_package:build_pip_package && \ - mkdir -p /pip_pkg && \ +# Build and Install TensorFlow. +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ + LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ + bazel build -c opt \ + --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + --jobs=${TF_AVAILABLE_CPUS} \ + tensorflow/tools/pip_package:build_pip_package && \ + mkdir /pip_pkg && \ bazel-bin/tensorflow/tools/pip_package/build_pip_package /pip_pkg +# Clean up pip wheel and Bazel cache when done. RUN pip --no-cache-dir install --upgrade /pip_pkg/tensorflow-*.whl && \ + rm -rf /pip_pkg && \ + rm -rf /root/.cache WORKDIR /root diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index da83a30058..0571dd7391 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -1,6 +1,6 @@ FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04 -MAINTAINER Craig Citro +LABEL maintainer="Craig Citro " # Pick up some TF dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/tensorflow/tools/docker/README.md b/tensorflow/tools/docker/README.md index 3780bde2be..2e5a0038ed 100644 --- a/tensorflow/tools/docker/README.md +++ b/tensorflow/tools/docker/README.md @@ -41,6 +41,7 @@ Note: If you would have a problem running nvidia-docker you may try the old meth we have used. But it is not recommended. If you find a bug in nvidia-docker, please report it there and try using nvidia-docker as described above. + $ # The old, not recommended way to run docker with gpu support: $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}') $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index c05d39e942..4f0de8f768 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -29,14 +29,14 @@ from setuptools.dist import Distribution # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.3.0' +_VERSION = '1.4.0-rc0' REQUIRED_PACKAGES = [ 'enum34 >= 1.1.6', 'numpy >= 1.12.1', 'six >= 1.10.0', 'protobuf >= 3.4.0', - 'tensorflow-tensorboard >= 0.1.0, < 0.2.0', + 'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0', ] project_name = 'tensorflow' @@ -67,6 +67,7 @@ if sys.version_info < (3, 4): # pylint: disable=line-too-long CONSOLE_SCRIPTS = [ + 'freeze_graph = tensorflow.python.tools.freeze_graph:main', 'saved_model_cli = tensorflow.python.tools.saved_model_cli:main', # We need to keep the TensorBoard command, even though the console script # is now declared by the tensorboard pip package. If we remove the diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 4d577fc246..a14469a0be 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -590,16 +590,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""): actual = "@jsoncpp_git//:jsoncpp", ) - patched_http_archive( + native.http_archive( name = "boringssl", urls = [ - "https://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz", - # "https://github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz", # 2017-07-07 + "https://mirror.bazel.build/github.com/google/boringssl/archive/a0fb951d2a26a8ee746b52f3ba81ab011a0af778.tar.gz", ], - sha256 = "02f5950f93c4fd3691771c07c9d04cf2999ab01383ff99da345249e93b0fcfb2", - strip_prefix = "boringssl-e3860009a091cd1bd2bc189cdbc3c6d095abde84", - # Add patch to boringssl code to support s390x - patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")), + sha256 = "524ba98a56300149696481b4cb9ddebd0c7b7ac9b9f6edee81da2d2d7e5d2bb3", + strip_prefix = "boringssl-a0fb951d2a26a8ee746b52f3ba81ab011a0af778", ) native.new_http_archive( @@ -701,6 +698,31 @@ def tf_workspace(path_prefix="", tf_repo_name=""): repository = tf_repo_name, ) + java_import_external( + name = "com_google_testing_compile", + jar_sha256 = "edc180fdcd9f740240da1a7a45673f46f59c5578d8cd3fbc912161f74b5aebb8", + jar_urls = [ + "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar", + "http://repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar", + "http://maven.ibiblio.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar", + ], + licenses = ["notice"], # New BSD License + testonly_ = True, + deps = ["@com_google_guava", "@com_google_truth"], + ) + + java_import_external( + name = "com_google_truth", + jar_sha256 = "032eddc69652b0a1f8d458f999b4a9534965c646b8b5de0eba48ee69407051df", + jar_urls = [ + "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/truth/truth/0.32/truth-0.32.jar", + "http://repo1.maven.org/maven2/com/google/truth/truth/0.32/truth-0.32.jar", + ], + licenses = ["notice"], # Apache 2.0 + testonly_ = True, + deps = ["@com_google_guava"], + ) + native.new_http_archive( name = "com_google_pprof", urls = [ @@ -715,11 +737,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "cub_archive", urls = [ - "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip", - # "https://github.com/NVlabs/cub/archive/1.7.3.zip", + "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip", + # "https://github.com/NVlabs/cub/archive/1.7.4.zip", ], - sha256 = "b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe", - strip_prefix = "cub-1.7.3", + sha256 = "20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31", + strip_prefix = "cub-1.7.4", build_file = str(Label("//third_party:cub.BUILD")), ) diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD index 38b7e0e543..9d8e7946cd 100644 --- a/third_party/aws.BUILD +++ b/third_party/aws.BUILD @@ -18,6 +18,7 @@ cc_library( "@%ws%//tensorflow:darwin": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), + "//conditions:default": [], }) + glob([ "aws-cpp-sdk-core/include/**/*.h", "aws-cpp-sdk-core/source/*.cpp", diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h index 078be83e0d..c210b1712c 100644 --- a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h +++ b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h @@ -1,6 +1,35 @@ #ifndef THIRD_PARTY_EIGEN3_UNSUPPORTED_EIGEN_CXX11_SRC_FIXEDPOINT_PACKETMATHAVX2_H_ #define THIRD_PARTY_EIGEN3_UNSUPPORTED_EIGEN_CXX11_SRC_FIXEDPOINT_PACKETMATHAVX2_H_ +#ifdef _MSC_VER + +#include +#include +#include + +#endif + +inline int _mm256_extract_epi16_N0(const __m256i X) +{ + return _mm_extract_epi16(_mm256_extractf128_si256(X, 0 >> 3), 0 % 8); +} + +inline int _mm256_extract_epi16_N1(const __m256i X) +{ + return _mm_extract_epi16(_mm256_extractf128_si256(X, 1 >> 3), 1 % 8); +} + +inline int _mm256_extract_epi8_N0(const __m256i X) +{ + return _mm_extract_epi8(_mm256_extractf128_si256((X), 0 >> 4), 0 % 16); +} + +inline int _mm256_extract_epi8_N1(const __m256i X) +{ + return _mm_extract_epi8(_mm256_extractf128_si256((X), 1 >> 4), 1 % 16); +} + + namespace Eigen { namespace internal { @@ -271,15 +300,15 @@ EIGEN_STRONG_INLINE QInt32 pfirst(const Packet8q32i& a) { } template <> EIGEN_STRONG_INLINE QInt16 pfirst(const Packet16q16i& a) { - return _mm256_extract_epi16(a.val, 0); + return _mm256_extract_epi16_N0(a.val); } template <> EIGEN_STRONG_INLINE QUInt8 pfirst(const Packet32q8u& a) { - return static_cast(_mm256_extract_epi8(a.val, 0)); + return static_cast(_mm256_extract_epi8_N0(a.val)); } template <> EIGEN_STRONG_INLINE QInt8 pfirst(const Packet32q8i& a) { - return _mm256_extract_epi8(a.val, 0); + return _mm256_extract_epi8_N0(a.val); } // Initialize to constant value. @@ -391,7 +420,7 @@ EIGEN_STRONG_INLINE QInt16 predux_min(const Packet16q16i& a) { tmp = _mm256_min_epi16(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2))); tmp = _mm256_min_epi16(tmp, _mm256_shuffle_epi32(tmp, 1)); - return std::min(_mm256_extract_epi16(tmp, 0), _mm256_extract_epi16(tmp, 1)); + return std::min(_mm256_extract_epi16_N0(tmp), _mm256_extract_epi16_N1(tmp)); } template <> EIGEN_STRONG_INLINE QInt16 predux_max(const Packet16q16i& a) { @@ -399,7 +428,7 @@ EIGEN_STRONG_INLINE QInt16 predux_max(const Packet16q16i& a) { tmp = _mm256_max_epi16(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2))); tmp = _mm256_max_epi16(tmp, _mm256_shuffle_epi32(tmp, 1)); - return std::max(_mm256_extract_epi16(tmp, 0), _mm256_extract_epi16(tmp, 1)); + return std::max(_mm256_extract_epi16_N0(tmp), _mm256_extract_epi16_N1(tmp)); } template <> @@ -410,8 +439,8 @@ EIGEN_STRONG_INLINE QUInt8 predux_min(const Packet32q8u& a) { tmp = _mm256_min_epu8(tmp, _mm256_shuffle_epi32(tmp, 1)); tmp = _mm256_min_epu8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2))); - return std::min(static_cast(_mm256_extract_epi8(tmp, 0)), - static_cast(_mm256_extract_epi8(tmp, 1))); + return std::min(static_cast(_mm256_extract_epi8_N0(tmp)), + static_cast(_mm256_extract_epi8_N1(tmp))); } template <> EIGEN_STRONG_INLINE QUInt8 predux_max(const Packet32q8u& a) { @@ -421,8 +450,8 @@ EIGEN_STRONG_INLINE QUInt8 predux_max(const Packet32q8u& a) { tmp = _mm256_max_epu8(tmp, _mm256_shuffle_epi32(tmp, 1)); tmp = _mm256_max_epu8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2))); - return std::max(static_cast(_mm256_extract_epi8(tmp, 0)), - static_cast(_mm256_extract_epi8(tmp, 1))); + return std::max(static_cast(_mm256_extract_epi8_N0(tmp)), + static_cast(_mm256_extract_epi8_N1(tmp))); } template <> @@ -431,7 +460,7 @@ EIGEN_STRONG_INLINE QInt8 predux_min(const Packet32q8i& a) { tmp = _mm256_min_epi8(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2))); tmp = _mm256_min_epi8(tmp, _mm256_shuffle_epi32(tmp, 1)); tmp = _mm256_min_epi8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2))); - return std::min(_mm256_extract_epi8(tmp, 0), _mm256_extract_epi8(tmp, 1)); + return std::min(_mm256_extract_epi8_N0(tmp), _mm256_extract_epi8_N1(tmp)); } template <> EIGEN_STRONG_INLINE QInt8 predux_max(const Packet32q8i& a) { @@ -439,7 +468,7 @@ EIGEN_STRONG_INLINE QInt8 predux_max(const Packet32q8i& a) { tmp = _mm256_max_epi8(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2))); tmp = _mm256_max_epi8(tmp, _mm256_shuffle_epi32(tmp, 1)); tmp = _mm256_max_epi8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2))); - return std::max(_mm256_extract_epi8(tmp, 0), _mm256_extract_epi8(tmp, 1)); + return std::max(_mm256_extract_epi8_N0(tmp), _mm256_extract_epi8_N1(tmp)); } // Vectorized scaling of Packet32q8i by float. diff --git a/third_party/toolchains/cpus/arm/CROSSTOOL.tpl b/third_party/toolchains/cpus/arm/CROSSTOOL.tpl index ad7f5596d0..f0e17d1fe0 100644 --- a/third_party/toolchains/cpus/arm/CROSSTOOL.tpl +++ b/third_party/toolchains/cpus/arm/CROSSTOOL.tpl @@ -87,7 +87,7 @@ toolchain { cxx_flag: "-isystem" cxx_flag: "/usr/include/arm-linux-gnueabihf" cxx_flag: "-isystem" - cxx_flag: "/usr/include/python2.7" + cxx_flag: "%{PYTHON_INCLUDE_PATH}%" cxx_flag: "-isystem" cxx_flag: "/usr/include/" linker_flag: "-lstdc++" diff --git a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl index 5eb3b7bb1c..ab6eac115c 100644 --- a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl +++ b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl @@ -11,9 +11,20 @@ def _tpl(repository_ctx, tpl, substitutions={}, out=None): def _arm_compiler_configure_impl(repository_ctx): + # We need to find a cross-compilation include directory for Python, so look + # for an environment variable. Be warned, this crosstool template is only + # regenerated on the first run of Bazel, so if you change the variable after + # it may not be reflected in later builds. Doing a shutdown and clean of Bazel + # doesn't fix this, you'll need to delete the generated file at something like: + # external/local_config_arm_compiler/CROSSTOOL in your Bazel install. + if "CROSSTOOL_PYTHON_INCLUDE_PATH" in repository_ctx.os.environ: + python_include_path = repository_ctx.os.environ["CROSSTOOL_PYTHON_INCLUDE_PATH"] + else: + python_include_path = "/usr/include/python2.7" _tpl(repository_ctx, "CROSSTOOL", { "%{ARM_COMPILER_PATH}%": str(repository_ctx.path( repository_ctx.attr.remote_config_repo)), + "%{PYTHON_INCLUDE_PATH}%": python_include_path, }) repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD") -- GitLab From 3a103d98d5830de5bde7c0713a247137f7f37804 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 24 Oct 2017 19:49:49 -0700 Subject: [PATCH 336/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173347713 --- .../core/ops/compat/ops_history.v1.pbtxt | 400 ++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 171 +++++++- 2 files changed, 563 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 92037c1997..e6aeb35e02 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -39,6 +39,54 @@ op { } } } +op { + name: "AccumulateNV2" + input_arg { + name: "inputs" + type_attr: "T" + number_attr: "N" + } + output_arg { + name: "sum" + type_attr: "T" + } + attr { + name: "N" + type: "int" + has_minimum: true + minimum: 1 + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "shape" + type: "shape" + } + is_aggregate: true + is_commutative: true +} op { name: "AccumulatorApplyGradient" input_arg { @@ -7653,6 +7701,65 @@ op { } } } +op { + name: "CropAndResize" + input_arg { + name: "image" + type_attr: "T" + } + input_arg { + name: "boxes" + type: DT_FLOAT + } + input_arg { + name: "box_ind" + type: DT_INT32 + } + input_arg { + name: "crop_size" + type: DT_INT32 + } + output_arg { + name: "crops" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "method" + type: "string" + default_value { + s: "bilinear" + } + allowed_values { + list { + s: "bilinear" + } + } + } + attr { + name: "extrapolation_value" + type: "float" + default_value { + f: 0 + } + } +} op { name: "CropAndResizeGradBoxes" input_arg { @@ -7704,6 +7811,58 @@ op { } } } +op { + name: "CropAndResizeGradBoxes" + input_arg { + name: "grads" + type: DT_FLOAT + } + input_arg { + name: "image" + type_attr: "T" + } + input_arg { + name: "boxes" + type: DT_FLOAT + } + input_arg { + name: "box_ind" + type: DT_INT32 + } + output_arg { + name: "output" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "method" + type: "string" + default_value { + s: "bilinear" + } + allowed_values { + list { + s: "bilinear" + } + } + } +} op { name: "CropAndResizeGradImage" input_arg { @@ -13533,6 +13692,50 @@ op { } is_stateful: true } +op { + name: "HistogramFixedWidth" + input_arg { + name: "values" + type_attr: "T" + } + input_arg { + name: "value_range" + type_attr: "T" + } + input_arg { + name: "nbins" + type: DT_INT32 + } + output_arg { + name: "out" + type_attr: "dtype" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "dtype" + type: "type" + default_value { + type: DT_INT32 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "HistogramSummary" input_arg { @@ -19710,6 +19913,47 @@ op { } is_commutative: true } +op { + name: "NthElement" + input_arg { + name: "input" + type_attr: "T" + } + input_arg { + name: "n" + type: DT_INT32 + } + output_arg { + name: "values" + type_attr: "T" + } + attr { + name: "reverse" + type: "bool" + default_value { + b: false + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } +} op { name: "OneHot" input_arg { @@ -25190,6 +25434,45 @@ op { } } } +op { + name: "ResizeArea" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} op { name: "ResizeBicubic" input_arg { @@ -25228,6 +25511,45 @@ op { } } } +op { + name: "ResizeBicubic" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} op { name: "ResizeBicubicGrad" input_arg { @@ -25298,6 +25620,45 @@ op { } } } +op { + name: "ResizeBilinear" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} op { name: "ResizeBilinearGrad" input_arg { @@ -25369,6 +25730,45 @@ op { } } } +op { + name: "ResizeNearestNeighbor" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} op { name: "ResizeNearestNeighborGrad" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index c037c99c19..a6886e465d 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -44,6 +44,58 @@ op { summary: "Computes the absolute value of a tensor." description: "Given a tensor `x`, this operation returns a tensor containing the absolute\nvalue of each element in `x`. For example, if x is an input element and y is\nan output element, this operation computes \\\\(y = |x|\\\\)." } +op { + name: "AccumulateNV2" + input_arg { + name: "inputs" + description: "A list of `Tensor` objects, each with same shape and type." + type_attr: "T" + number_attr: "N" + } + output_arg { + name: "sum" + type_attr: "T" + } + attr { + name: "N" + type: "int" + has_minimum: true + minimum: 1 + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "shape" + type: "shape" + description: "Shape of elements of `inputs`." + } + summary: "Returns the element-wise sum of a list of tensors." + description: "`tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not\nwait for all of its inputs to be ready before beginning to sum. This can\nsave memory if inputs are ready at different times, since minimum temporary\nstorage is proportional to the output size rather than the inputs size.\n\nUnlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.\n\nReturns a `Tensor` of same shape and type as the elements of `inputs`." + is_aggregate: true + is_commutative: true +} op { name: "AccumulatorApplyGradient" input_arg { @@ -5639,6 +5691,7 @@ op { allowed_values { list { type: DT_UINT8 + type: DT_UINT16 type: DT_INT8 type: DT_INT16 type: DT_INT32 @@ -5706,6 +5759,7 @@ op { allowed_values { list { type: DT_UINT8 + type: DT_UINT16 type: DT_INT8 type: DT_INT16 type: DT_INT32 @@ -7086,7 +7140,7 @@ op { name: "Diag" input_arg { name: "diagonal" - description: "Rank k tensor where k is at most 3." + description: "Rank k tensor where k is at most 1." type_attr: "T" } output_arg { @@ -7114,7 +7168,7 @@ op { name: "DiagPart" input_arg { name: "input" - description: "Rank k tensor where k is 2, 4, or 6." + description: "Rank k tensor where k is even and not zero." type_attr: "T" } output_arg { @@ -10300,6 +10354,56 @@ op { description: "This op creates a hash table, specifying the type of its keys and values.\nBefore using the table you will have to initialize it. After initialization the\ntable will be immutable." is_stateful: true } +op { + name: "HistogramFixedWidth" + input_arg { + name: "values" + description: "Numeric `Tensor`." + type_attr: "T" + } + input_arg { + name: "value_range" + description: "Shape [2] `Tensor` of same `dtype` as `values`.\nvalues <= value_range[0] will be mapped to hist[0],\nvalues >= value_range[1] will be mapped to hist[-1]." + type_attr: "T" + } + input_arg { + name: "nbins" + description: "Scalar `int32 Tensor`. Number of histogram bins." + type: DT_INT32 + } + output_arg { + name: "out" + description: "A 1-D `Tensor` holding histogram of values." + type_attr: "dtype" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "dtype" + type: "type" + default_value { + type: DT_INT32 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + summary: "Return histogram of values." + description: "Given the tensor `values`, this operation returns a rank 1 histogram counting\nthe number of entries in `values` that fall into every bin. The bins are\nequal width and determined by the arguments `value_range` and `nbins`.\n\n```python\n# Bins will be: (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)\nnbins = 5\nvalue_range = [0.0, 5.0]\nnew_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]\n\nwith tf.get_default_session() as sess:\n hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)\n variables.global_variables_initializer().run()\n sess.run(hist) => [2, 1, 1, 0, 2]\n```" +} op { name: "HistogramSummary" input_arg { @@ -15237,6 +15341,53 @@ op { description: "*NOTE*: `NotEqual` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" is_commutative: true } +op { + name: "NthElement" + input_arg { + name: "input" + description: "1-D or higher with last dimension at least `n+1`." + type_attr: "T" + } + input_arg { + name: "n" + description: "0-D. Position of sorted vector to select along the last dimension (along\neach row for matrices). Valid range of n is `[0, input.shape[:-1])`" + type: DT_INT32 + } + output_arg { + name: "values" + description: "The `n`-th order statistic along each last dimensional slice." + type_attr: "T" + } + attr { + name: "reverse" + type: "bool" + default_value { + b: false + } + description: "When set to True, find the nth-largest value in the vector and vice\nversa." + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + summary: "Finds values of the `n`-th order statistic for the last dmension." + description: "If the input is a vector (rank-1), finds the entries which is the nth-smallest\nvalue in the vector and outputs their values as scalar tensor.\n\nFor matrices (resp. higher rank input), computes the entries which is the\nnth-smallest value in each row (resp. vector along the last dimension). Thus,\n\n values.shape = input.shape[:-1]" +} op { name: "OneHot" input_arg { @@ -20814,9 +20965,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -20858,9 +21010,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -20939,9 +21092,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -21021,9 +21175,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -23255,7 +23410,7 @@ op { } } summary: "Reverses specific dimensions of a tensor." - description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0, 1, 2, 3],\n# [ 4, 5, 6, 7],\n# [ 8, 9, 10, 11]],\n# [[12, 13, 14, 15],\n# [16, 17, 18, 19],\n# [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is -1\nreverse(t, dims) ==> [[[[ 3, 2, 1, 0],\n [ 7, 6, 5, 4],\n [ 11, 10, 9, 8]],\n [[15, 14, 13, 12],\n [19, 18, 17, 16],\n [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n [16, 17, 18, 19],\n [20, 21, 22, 23]\n [[ 0, 1, 2, 3],\n [ 4, 5, 6, 7],\n [ 8, 9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n [4, 5, 6, 7],\n [0, 1, 2, 3]]\n [[20, 21, 22, 23],\n [16, 17, 18, 19],\n [12, 13, 14, 15]]]]\n```" + description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0, 1, 2, 3],\n# [ 4, 5, 6, 7],\n# [ 8, 9, 10, 11]],\n# [[12, 13, 14, 15],\n# [16, 17, 18, 19],\n# [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is [-1]\nreverse(t, dims) ==> [[[[ 3, 2, 1, 0],\n [ 7, 6, 5, 4],\n [ 11, 10, 9, 8]],\n [[15, 14, 13, 12],\n [19, 18, 17, 16],\n [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n [16, 17, 18, 19],\n [20, 21, 22, 23]\n [[ 0, 1, 2, 3],\n [ 4, 5, 6, 7],\n [ 8, 9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n [4, 5, 6, 7],\n [0, 1, 2, 3]]\n [[20, 21, 22, 23],\n [16, 17, 18, 19],\n [12, 13, 14, 15]]]]\n```" } op { name: "RightShift" @@ -32065,7 +32220,7 @@ op { } } summary: "Returns x / y element-wise for integer types." - description: "Truncation designates that negative numbers will round fractional quantities\ntoward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different\nthan Python semantics. See `FloorDiv` for a division function that matches\nPython Semantics.\n\n*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" + description: "Truncation designates that negative numbers will round fractional quantities\ntoward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different\nthan Python semantics. See `FloorDiv` for a division function that matches\nPython Semantics.\n\n*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" } op { name: "TruncateMod" -- GitLab From 16953025097793d9748099ebf4296edca04a5366 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 24 Oct 2017 19:54:58 -0700 Subject: [PATCH 337/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173347919 --- tensorflow/go/op/wrappers.go | 491 ++++++++++++++++++++++------------- 1 file changed, 313 insertions(+), 178 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index b3b317013f..958ce6d040 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -1849,7 +1849,7 @@ func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf // ``` // // Arguments: -// input: Rank k tensor where k is 2, 4, or 6. +// input: Rank k tensor where k is even and not zero. // // Returns The extracted diagonal. func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) { @@ -4140,7 +4140,7 @@ func TensorArraySplitV3(scope *Scope, handle tf.Output, value tf.Output, lengths // ``` // // Arguments: -// diagonal: Rank k tensor where k is at most 3. +// diagonal: Rank k tensor where k is at most 1. func Diag(scope *Scope, diagonal tf.Output) (output tf.Output) { if scope.Err() != nil { return @@ -7833,6 +7833,113 @@ func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer return op.Output(0) } +// Adjust the saturation of one or more images. +// +// `images` is a tensor of at least 3 dimensions. The last dimension is +// interpretted as channels, and must be three. +// +// The input image is considered in the RGB colorspace. Conceptually, the RGB +// colors are first mapped into HSV. A scale is then applied all the saturation +// values, and then remapped back to RGB colorspace. +// +// Arguments: +// images: Images to adjust. At least 3-D. +// scale: A float scale to add to the saturation. +// +// Returns The hue-adjusted image or images. +func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "AdjustSaturation", + Input: []tf.Input{ + images, scale, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2. +type SelfAdjointEigV2Attr func(optionalAttr) + +// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value. +// +// value: If `True` then eigenvectors will be computed and returned in `v`. +// Otherwise, only the eigenvalues will be computed. +// If not specified, defaults to true +func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr { + return func(m optionalAttr) { + m["compute_v"] = value + } +} + +// Computes the eigen decomposition of one or more square self-adjoint matrices. +// +// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in +// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. +// +// ```python +// # a is a tensor. +// # e is a tensor of eigenvalues. +// # v is a tensor of eigenvectors. +// e, v = self_adjoint_eig(a) +// e = self_adjoint_eig(a, compute_v=False) +// ``` +// +// Arguments: +// input: `Tensor` input of shape `[N, N]`. +// +// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`. +func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "SelfAdjointEigV2", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + +// Computes the Eigen Decomposition of a batch of square self-adjoint matrices. +// +// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead. +// +// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices, with the same constraints as the single matrix +// SelfAdjointEig. +// +// The result is a [..., M+1, M] matrix with [..., 0,:] containing the +// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. +// +// Arguments: +// input: Shape is `[..., M, M]`. +// +// Returns Shape is `[..., M+1, M]`. +func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SelfAdjointEig", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad. type FusedBatchNormGradAttr func(optionalAttr) @@ -10567,6 +10674,36 @@ func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...Ran return op.Output(0) } +// Returns the element-wise sum of a list of tensors. +// +// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not +// wait for all of its inputs to be ready before beginning to sum. This can +// save memory if inputs are ready at different times, since minimum temporary +// storage is proportional to the output size rather than the inputs size. +// +// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable. +// +// Returns a `Tensor` of same shape and type as the elements of `inputs`. +// +// Arguments: +// inputs: A list of `Tensor` objects, each with same shape and type. +// shape: Shape of elements of `inputs`. +func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"shape": shape} + opspec := tf.OpSpec{ + Type: "AccumulateNV2", + Input: []tf.Input{ + tf.OutputList(inputs), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Computes the gradient for the inverse of `x` wrt its input. // // Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` @@ -13641,35 +13778,6 @@ func Rint(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } -// Computes the Eigen Decomposition of a batch of square self-adjoint matrices. -// -// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead. -// -// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices, with the same constraints as the single matrix -// SelfAdjointEig. -// -// The result is a [..., M+1, M] matrix with [..., 0,:] containing the -// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. -// -// Arguments: -// input: Shape is `[..., M, M]`. -// -// Returns Shape is `[..., M+1, M]`. -func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SelfAdjointEig", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // QuantizeV2Attr is an optional argument to QuantizeV2. type QuantizeV2Attr func(optionalAttr) @@ -14310,7 +14418,7 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max // # [20, 21, 22, 23]]]] // # tensor 't' shape is [1, 2, 3, 4] // -// # 'dims' is [3] or 'dims' is -1 +// # 'dims' is [3] or 'dims' is [-1] // reverse(t, dims) ==> [[[[ 3, 2, 1, 0], // [ 7, 6, 5, 4], // [ 11, 10, 9, 8]], @@ -14534,6 +14642,73 @@ func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, fil return op.Output(0) } +// MatrixSolveAttr is an optional argument to MatrixSolve. +type MatrixSolveAttr func(optionalAttr) + +// MatrixSolveAdjoint sets the optional adjoint attribute to value. +// +// value: Boolean indicating whether to solve with `matrix` or its (block-wise) +// adjoint. +// If not specified, defaults to false +func MatrixSolveAdjoint(value bool) MatrixSolveAttr { + return func(m optionalAttr) { + m["adjoint"] = value + } +} + +// Solves systems of linear equations. +// +// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is +// a tensor shape `[..., M, K]`. If `adjoint` is `False` then each output matrix +// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`. +// If `adjoint` is `True` then each output matrix satisfies +// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`. +// +// Arguments: +// matrix: Shape is `[..., M, M]`. +// rhs: Shape is `[..., M, K]`. +// +// Returns Shape is `[..., M, K]`. +func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MatrixSolve", + Input: []tf.Input{ + matrix, rhs, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Transforms a Tensor into a serialized TensorProto proto. +// +// Arguments: +// tensor: A Tensor of type `T`. +// +// Returns A serialized TensorProto proto of the input tensor. +func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SerializeTensor", + Input: []tf.Input{ + tensor, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Get the value of the tensor specified by its handle. // // Arguments: @@ -15915,7 +16090,7 @@ func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and // Returns x / y element-wise for integer types. // // Truncation designates that negative numbers will round fractional quantities -// toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different +// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different // than Python semantics. See `FloorDiv` for a division function that matches // Python Semantics. // @@ -20558,84 +20733,6 @@ func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output return op.Output(0) } -// Adjust the saturation of one or more images. -// -// `images` is a tensor of at least 3 dimensions. The last dimension is -// interpretted as channels, and must be three. -// -// The input image is considered in the RGB colorspace. Conceptually, the RGB -// colors are first mapped into HSV. A scale is then applied all the saturation -// values, and then remapped back to RGB colorspace. -// -// Arguments: -// images: Images to adjust. At least 3-D. -// scale: A float scale to add to the saturation. -// -// Returns The hue-adjusted image or images. -func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "AdjustSaturation", - Input: []tf.Input{ - images, scale, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2. -type SelfAdjointEigV2Attr func(optionalAttr) - -// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value. -// -// value: If `True` then eigenvectors will be computed and returned in `v`. -// Otherwise, only the eigenvalues will be computed. -// If not specified, defaults to true -func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr { - return func(m optionalAttr) { - m["compute_v"] = value - } -} - -// Computes the eigen decomposition of one or more square self-adjoint matrices. -// -// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in -// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. -// -// ```python -// # a is a tensor. -// # e is a tensor of eigenvalues. -// # v is a tensor of eigenvectors. -// e, v = self_adjoint_eig(a) -// e = self_adjoint_eig(a, compute_v=False) -// ``` -// -// Arguments: -// input: `Tensor` input of shape `[N, N]`. -// -// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`. -func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "SelfAdjointEigV2", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - // Computes second-order gradients of the maxpooling function. // // Arguments: @@ -22256,6 +22353,62 @@ func QuantizedBiasAdd(scope *Scope, input tf.Output, bias tf.Output, min_input t return op.Output(0), op.Output(1), op.Output(2) } +// HistogramFixedWidthAttr is an optional argument to HistogramFixedWidth. +type HistogramFixedWidthAttr func(optionalAttr) + +// HistogramFixedWidthDtype sets the optional dtype attribute to value. +// If not specified, defaults to DT_INT32 +func HistogramFixedWidthDtype(value tf.DataType) HistogramFixedWidthAttr { + return func(m optionalAttr) { + m["dtype"] = value + } +} + +// Return histogram of values. +// +// Given the tensor `values`, this operation returns a rank 1 histogram counting +// the number of entries in `values` that fall into every bin. The bins are +// equal width and determined by the arguments `value_range` and `nbins`. +// +// ```python +// # Bins will be: (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf) +// nbins = 5 +// value_range = [0.0, 5.0] +// new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] +// +// with tf.get_default_session() as sess: +// hist = tf.histogram_fixed_width(new_values, value_range, nbins=5) +// variables.global_variables_initializer().run() +// sess.run(hist) => [2, 1, 1, 0, 2] +// ``` +// +// Arguments: +// values: Numeric `Tensor`. +// value_range: Shape [2] `Tensor` of same `dtype` as `values`. +// values <= value_range[0] will be mapped to hist[0], +// values >= value_range[1] will be mapped to hist[-1]. +// nbins: Scalar `int32 Tensor`. Number of histogram bins. +// +// Returns A 1-D `Tensor` holding histogram of values. +func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output, nbins tf.Output, optional ...HistogramFixedWidthAttr) (out tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "HistogramFixedWidth", + Input: []tf.Input{ + values, value_range, nbins, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Quantized Batch normalization. // // This op is deprecated and will be removed in the future. Prefer @@ -23714,6 +23867,55 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu return op.Output(0) } +// NthElementAttr is an optional argument to NthElement. +type NthElementAttr func(optionalAttr) + +// NthElementReverse sets the optional reverse attribute to value. +// +// value: When set to True, find the nth-largest value in the vector and vice +// versa. +// If not specified, defaults to false +func NthElementReverse(value bool) NthElementAttr { + return func(m optionalAttr) { + m["reverse"] = value + } +} + +// Finds values of the `n`-th order statistic for the last dmension. +// +// If the input is a vector (rank-1), finds the entries which is the nth-smallest +// value in the vector and outputs their values as scalar tensor. +// +// For matrices (resp. higher rank input), computes the entries which is the +// nth-smallest value in each row (resp. vector along the last dimension). Thus, +// +// values.shape = input.shape[:-1] +// +// Arguments: +// input: 1-D or higher with last dimension at least `n+1`. +// n: 0-D. Position of sorted vector to select along the last dimension (along +// each row for matrices). Valid range of n is `[0, input.shape[:-1])` +// +// Returns The `n`-th order statistic along each last dimensional slice. +func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "NthElement", + Input: []tf.Input{ + input, n, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Computes asin of x element-wise. func Asin(scope *Scope, x tf.Output) (y tf.Output) { if scope.Err() != nil { @@ -26817,70 +27019,3 @@ func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) op := scope.AddOperation(opspec) return op.Output(0) } - -// Transforms a Tensor into a serialized TensorProto proto. -// -// Arguments: -// tensor: A Tensor of type `T`. -// -// Returns A serialized TensorProto proto of the input tensor. -func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SerializeTensor", - Input: []tf.Input{ - tensor, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// MatrixSolveAttr is an optional argument to MatrixSolve. -type MatrixSolveAttr func(optionalAttr) - -// MatrixSolveAdjoint sets the optional adjoint attribute to value. -// -// value: Boolean indicating whether to solve with `matrix` or its (block-wise) -// adjoint. -// If not specified, defaults to false -func MatrixSolveAdjoint(value bool) MatrixSolveAttr { - return func(m optionalAttr) { - m["adjoint"] = value - } -} - -// Solves systems of linear equations. -// -// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is -// a tensor shape `[..., M, K]`. If `adjoint` is `False` then each output matrix -// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`. -// If `adjoint` is `True` then each output matrix satisfies -// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`. -// -// Arguments: -// matrix: Shape is `[..., M, M]`. -// rhs: Shape is `[..., M, K]`. -// -// Returns Shape is `[..., M, K]`. -func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MatrixSolve", - Input: []tf.Input{ - matrix, rhs, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} -- GitLab From 557b0b27edff763c165ad59d10d49da8bccbec4f Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Tue, 24 Oct 2017 20:05:39 -0700 Subject: [PATCH 338/573] Make HloRunner methods return StatusOr. Also move templated method definition of Execute into the header file. PiperOrigin-RevId: 173348703 --- tensorflow/compiler/xla/service/hlo_runner.cc | 40 +++++-------------- tensorflow/compiler/xla/service/hlo_runner.h | 24 ++++++++--- .../compiler/xla/tests/hlo_test_base.cc | 6 +-- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index d5d7042a02..9fdda38d2d 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -29,7 +29,6 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/transfer_manager.h" #include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/common_runtime/eigen_thread_pool.h" @@ -133,7 +132,8 @@ StatusOr HloRunner::Execute( return result; } -se::DeviceMemoryBase HloRunner::TransferToDevice(const Literal& literal) { +StatusOr HloRunner::TransferToDevice( + const Literal& literal) { // Allocate memory on the device using the stream executor. int64 allocation_size = backend().transfer_manager()->GetByteSizeRequirement(literal.shape()); @@ -142,52 +142,30 @@ se::DeviceMemoryBase HloRunner::TransferToDevice(const Literal& literal) { allocation_size); allocations_.push_back(allocation); - TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice( + TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( backend().default_stream_executor(), literal, &allocation)); return allocation; } -std::unique_ptr HloRunner::TransferFromDevice( +StatusOr> HloRunner::TransferFromDevice( const Shape& shape, se::DeviceMemoryBase device_base) { auto literal = MakeUnique(); - TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice( + TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralFromDevice( backend().default_stream_executor(), device_base, shape, shape, literal.get())); - return literal; + return std::move(literal); } -std::unique_ptr HloRunner::ExecuteAndTransfer( +StatusOr> HloRunner::ExecuteAndTransfer( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments) { Shape result_shape; - se::DeviceMemoryBase device_base = - Execute(std::move(module), arguments, &result_shape).ValueOrDie(); + TF_ASSIGN_OR_RETURN(se::DeviceMemoryBase device_base, + Execute(std::move(module), arguments, &result_shape)); return TransferFromDevice(result_shape, device_base); } -template <> -std::unique_ptr HloRunner::Execute( - std::unique_ptr module, - const tensorflow::gtl::ArraySlice>& literals) { - std::vector arguments; - for (const auto& literal : literals) { - arguments.push_back(TransferToDevice(*literal)); - } - return ExecuteAndTransfer(std::move(module), arguments); -} - -template <> -std::unique_ptr HloRunner::Execute( - std::unique_ptr module, - const tensorflow::gtl::ArraySlice& literals) { - std::vector arguments; - for (const auto& literal : literals) { - arguments.push_back(TransferToDevice(*literal)); - } - return ExecuteAndTransfer(std::move(module), arguments); -} - Backend& HloRunner::backend() { if (!backend_) { backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie(); diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h index d74a1b59a8..a4d7b653db 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.h +++ b/tensorflow/compiler/xla/service/hlo_runner.h @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/compiler.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" @@ -52,9 +53,9 @@ class HloRunner { // result as a Literal. The LiteralPtr type accepts Literal* or // std::unique_ptr. template - std::unique_ptr Execute( + StatusOr> Execute( std::unique_ptr module, - const tensorflow::gtl::ArraySlice& literals); + const tensorflow::gtl::ArraySlice literals); // Executes the given module and returns a global data handle. StatusOr Execute( @@ -64,16 +65,16 @@ class HloRunner { Shape* result_shape); // Transfers the given literal to the device and returns the data handle. - perftools::gputools::DeviceMemoryBase TransferToDevice( + StatusOr TransferToDevice( const Literal& literal); // Transfers the array referred to by the given handle from the device and // returns as a Literal. - std::unique_ptr TransferFromDevice( + StatusOr> TransferFromDevice( const Shape& shape, perftools::gputools::DeviceMemoryBase device_base); // Executes the given module and return the result as a Literal. - std::unique_ptr ExecuteAndTransfer( + StatusOr> ExecuteAndTransfer( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments); @@ -95,6 +96,19 @@ class HloRunner { std::unique_ptr backend_; }; +template +StatusOr> HloRunner::Execute( + std::unique_ptr module, + const tensorflow::gtl::ArraySlice literals) { + std::vector arguments; + for (const auto& literal : literals) { + TF_ASSIGN_OR_RETURN(perftools::gputools::DeviceMemoryBase argument, + TransferToDevice(*literal)); + arguments.push_back(argument); + } + return ExecuteAndTransfer(std::move(module), arguments); +} + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc index 3e244fbfd9..d73c05ff92 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.cc +++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc @@ -53,18 +53,18 @@ StatusOr HloTestBase::Execute( } se::DeviceMemoryBase HloTestBase::TransferToDevice(const Literal& literal) { - return runner_.TransferToDevice(literal); + return runner_.TransferToDevice(literal).ValueOrDie(); } std::unique_ptr HloTestBase::TransferFromDevice( const Shape& shape, se::DeviceMemoryBase device_base) { - return runner_.TransferFromDevice(shape, device_base); + return runner_.TransferFromDevice(shape, device_base).ValueOrDie(); } std::unique_ptr HloTestBase::ExecuteAndTransfer( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments) { - return runner_.ExecuteAndTransfer(std::move(module), arguments); + return runner_.ExecuteAndTransfer(std::move(module), arguments).ValueOrDie(); } Backend& HloTestBase::backend() { return runner_.backend(); } -- GitLab From 7bd701a29e958be1f836eabc498a67d742c35cdf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 24 Oct 2017 20:37:06 -0700 Subject: [PATCH 339/573] Support more instructions in Hlo parser. - while, tuple, send/recv, get-tuple-element, call. - "device=" Also, - Change HloModule::ToString to print computations in post order, so that a computation is defined before it's used. - Add % before computation name when it's used. PiperOrigin-RevId: 173350323 --- .../compiler/xla/service/hlo_computation.cc | 3 +- .../compiler/xla/service/hlo_instruction.cc | 17 +- .../xla/service/hlo_instruction_test.cc | 4 +- tensorflow/compiler/xla/service/hlo_module.cc | 14 +- .../compiler/xla/tools/parser/README.md | 24 +- .../compiler/xla/tools/parser/hlo_lexer.cc | 9 + .../compiler/xla/tools/parser/hlo_lexer.h | 9 +- .../compiler/xla/tools/parser/hlo_parser.cc | 323 ++++++++++++++---- .../xla/tools/parser/hlo_parser_test.cc | 117 ++++++- .../compiler/xla/tools/parser/hlo_token.h | 12 +- 10 files changed, 428 insertions(+), 104 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 51ead753f0..2285518a0e 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -379,7 +379,8 @@ string HloComputation::ToString(int nested_level) const { for (int i = 0; i < nested_level; i++) { s << " "; } - s << " " << instruction->ToString() << "\n"; + s << " " << (instruction == root_instruction_ ? "ROOT " : "") + << instruction->ToString() << "\n"; if (instruction->opcode() == HloOpcode::kFusion) { s << instruction->fused_instructions_computation()->ToString( nested_level + 1) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 0669a86863..8e52d131a6 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -1855,16 +1855,20 @@ std::vector HloInstruction::ExtraAttributesToString() const { } if (opcode() == HloOpcode::kWhile) { - extra.push_back(StrCat("condition=", while_condition()->name())); - extra.push_back(StrCat("body=", while_body()->name())); + extra.push_back(StrCat("condition=%", while_condition()->name())); + extra.push_back(StrCat("body=%", while_body()->name())); } else if (opcode() == HloOpcode::kSelectAndScatter) { - extra.push_back(StrCat("select=", select()->name())); - extra.push_back(StrCat("scatter=", scatter()->name())); + extra.push_back(StrCat("select=%", select()->name())); + extra.push_back(StrCat("scatter=%", scatter()->name())); + } else if (opcode() == HloOpcode::kCall || opcode() == HloOpcode::kMap || + opcode() == HloOpcode::kReduceWindow || + opcode() == HloOpcode::kReduce) { + extra.push_back(StrCat("to_apply=%", to_apply()->name())); } else if (!called_computations().empty()) { extra.push_back(StrCat( "calls=", Join(called_computations(), ", ", [](string* out, const HloComputation* computation) { - StrAppend(out, computation->name()); + StrAppend(out, "%", computation->name()); }))); } @@ -1875,6 +1879,9 @@ std::vector HloInstruction::ExtraAttributesToString() const { if (opcode() == HloOpcode::kGetTupleElement) { extra.push_back(StrCat("index=", tuple_index())); } + if (device_assignment_.has_device()) { + extra.push_back(StrCat("device=", device_assignment_.device())); + } if (!control_successors_.empty()) { extra.push_back(StrCat( "control-successors=", diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index cdafc05d8c..9affecae60 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -1203,13 +1203,13 @@ TEST_F(HloInstructionTest, Stringification) { EXPECT_EQ(fusion->ToString(false, false), "%fusion = f32[5,20]{1,0} fusion:kTransposeDot(f32[5,10]{1,0} %x, " - "f32[20,10]{1,0} %y), calls=fused_computation"); + "f32[20,10]{1,0} %y), calls=%fused_computation"); HloInstruction* loop = builder.AddInstruction( HloInstruction::CreateWhile(sout, computation, computation, x)); EXPECT_EQ(loop->ToString(false, false), "%while = f32[5,20]{1,0} while(f32[5,10]{1,0} %x), " - "condition=TransposeDot, body=TransposeDot"); + "condition=%TransposeDot, body=%TransposeDot"); } } // namespace diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 9d4a994838..f7990fa789 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -153,11 +153,17 @@ void HloModule::ReplaceComputations( string HloModule::ToString() const { std::ostringstream s; s << "HloModule " << name() << ":\n\n"; - s << "ENTRY " << entry_computation()->ToString() << "\n\n"; - for (const HloComputation* computation : MakeNonfusionComputations()) { - if (computation != entry_computation()) { - s << computation->ToString() << "\n\n"; + for (const HloComputation* computation : MakeComputationPostOrder()) { + // Fusion computations are emitted with their fusion instruction and + // therefore don't need to be emitted as a separate comptutation in the + // module. + if (computation->IsFusionComputation()) { + continue; + } + if (computation == entry_computation()) { + s << "ENTRY "; } + s << computation->ToString() << "\n\n"; } return s.str(); } diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md index a334bc2b29..2feaa49db8 100644 --- a/tensorflow/compiler/xla/tools/parser/README.md +++ b/tensorflow/compiler/xla/tools/parser/README.md @@ -1,16 +1,22 @@ # HloModule string syntax -TODO: Support subcomputations (for fusion, reduce, while, ...). +TODO: Support all subcomputations (for fusion, reduce, ...). -TODO: Support ops that require extra attributes, e.g. dimensions, strides. +TODO: Support all extra attributes, e.g. dimensions, strides. ```yacc hlo_module - : 'HloModule' name computation + : 'HloModule' name computations + ; + +computations + : computation + | computation computations ; computation : 'ENTRY' name param_list '->' shape instruction_list + | name param_list '->' shape instruction_list ; instruction_list @@ -21,7 +27,8 @@ instruction_list1 | instruction_list1 instruction ; instruction - : name '=' shape opcode operands + : 'ROOT' name '=' shape opcode operands extra_attributes + | name '=' shape opcode operands extra_attributes ; operands @@ -36,6 +43,15 @@ operand : shape name ; +extra_attributes + : /*empty*/ + | ',' extra_attribute + | ',' extra_attribute extra_attributes + ; +extra_attribute + : attribute_name attribute_value + ; + param_list : '(' param_list1 ')' ; diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc index 3e84ffcbd2..fba343de48 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc @@ -152,6 +152,7 @@ TokKind HloLexer::LexToken() { // name ::= [a-zA-Z_][a-zA-Z0-9_.-]*: // keyword ::= HloModule, ENTRY, ... // opcode ::= add, greater-than, ... +// attribute_name ::= condition, body, dimensions, ... TokKind HloLexer::LexIdentifier() { { auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); @@ -181,6 +182,13 @@ TokKind HloLexer::LexIdentifier() { return TokKind::kName; } + // If followed by '=', it's a attribute name. + if (PeekCurrentChar() == '=') { + str_val_.assign(token_start_, current_ptr_); + current_ptr_++; // skip '=' + return TokKind::kAttributeName; + } + StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_); // See if this is a keyword. @@ -195,6 +203,7 @@ TokKind HloLexer::LexIdentifier() { KEYWORD(false); KEYWORD(HloModule); KEYWORD(ENTRY); + KEYWORD(ROOT); #undef KEYWORD diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h index 20278fd6cd..433a3a3601 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h @@ -39,8 +39,13 @@ class HloLexer { TokKind Lex() { return current_kind_ = LexToken(); } TokKind GetKind() const { return current_kind_; } string GetStrVal() const { - CHECK(GetKind() == TokKind::kName); - return str_val_; + switch (GetKind()) { + case TokKind::kName: + case TokKind::kAttributeName: + return str_val_; + default: + LOG(FATAL) << "This token does not have string value"; + } } Shape GetShapeVal() const { CHECK(GetKind() == TokKind::kShape); diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc index 57700493e6..a075d9057f 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -44,14 +44,26 @@ class HloParser { private: // ParseXXX returns false if an error occurred. bool ParseHloModule(); + bool ParseComputations(); bool ParseComputation(); - bool ParseInstructionList(HloComputation::Builder* builder); - bool ParseInstruction(HloComputation::Builder* builder); + bool ParseInstructionList(HloComputation::Builder* builder, + string* root_name); + bool ParseInstruction(HloComputation::Builder* builder, string* root_name); bool ParseLiteral(std::unique_ptr* literal, const Shape& shape); + bool ParseOperands(std::vector* operands); + // Fill parsed operands into 'operands' and expect a certain number of + // operands. bool ParseOperands(std::vector* operands, const int expected_size); + + template + bool ParseExtraAttribute(T* value, const string& expected_attribute); + template + bool ParseAttributeValue(T* value); + bool ParseParamList(); bool ParseName(string* result); + bool ParseAttributeName(string* result); bool ParseShape(Shape* result); bool ParseOpcode(HloOpcode* result); bool ParseInt64(int64* result); @@ -69,10 +81,14 @@ class HloParser { // Adds the instruction to the pool. Returns false and emits an error if the // instruction already exists. bool AddInstruction(const string& name, HloInstruction* instruction); + // Adds the computation to the pool. Returns false and emits an error if the + // computation already exists. + bool AddComputation(const string& name, HloComputation* computation); // The map from the instruction name to the instruction. This does not own the // instructions. std::unordered_map instruction_pool_; + std::unordered_map computation_pool_; HloLexer lexer_; std::unique_ptr module_; @@ -90,7 +106,7 @@ bool HloParser::Run() { return ParseHloModule(); } -// ::= 'HloModule' name computation +// ::= 'HloModule' name computations bool HloParser::ParseHloModule() { if (lexer_.GetKind() != TokKind::kw_HloModule) { return TokenError("expects HloModule"); @@ -105,35 +121,63 @@ bool HloParser::ParseHloModule() { module_ = MakeUnique(name); - return ParseComputation(); + return ParseComputations(); } -// computation ::= 'ENTRY' name param_list '->' shape instruction_list +// computations ::= (computation)+ +bool HloParser::ParseComputations() { + do { + if (!ParseComputation()) { + return false; + } + } while (lexer_.GetKind() != TokKind::kEof); + return true; +} + +// computation ::= ('ENTRY')? name param_list '->' shape instruction_list bool HloParser::ParseComputation() { + const bool is_entry_computation = EatIfPresent(TokKind::kw_ENTRY); string name; - if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) { + if (!ParseName(&name)) { return false; } auto builder = MakeUnique(name); Shape shape; + string root_name; if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") || - !ParseShape(&shape) || !ParseInstructionList(builder.get())) { + !ParseShape(&shape) || !ParseInstructionList(builder.get(), &root_name)) { return false; } - module_->AddEntryComputation(builder->Build()); - return true; + + HloInstruction* root = + tensorflow::gtl::FindPtrOrNull(instruction_pool_, root_name); + // This means some instruction was marked as ROOT but we didn't find it in the + // pool, which should not happen. + if (!root_name.empty() && root == nullptr) { + LOG(FATAL) << "instruction " << root_name + << " was marked as ROOT but the parser has not seen it before"; + } + // Now root can be either an existing instruction or a nullptr. If it's a + // nullptr, the implementation of Builder will set the last instruction as + // root instruction. + HloComputation* computation = + is_entry_computation + ? module_->AddEntryComputation(builder->Build(root)) + : module_->AddEmbeddedComputation(builder->Build(root)); + return AddComputation(name, computation); } // instruction_list ::= '{' instruction_list1 '}' // instruction_list1 ::= (instruction)+ -bool HloParser::ParseInstructionList(HloComputation::Builder* builder) { +bool HloParser::ParseInstructionList(HloComputation::Builder* builder, + string* root_name) { if (!ParseToken(TokKind::kLbrace, "expects '{' at the beginning of instruction list.")) { return false; } do { - if (!ParseInstruction(builder)) { + if (!ParseInstruction(builder, root_name)) { return false; } } while (lexer_.GetKind() != TokKind::kRbrace); @@ -141,39 +185,47 @@ bool HloParser::ParseInstructionList(HloComputation::Builder* builder) { "expects '}' at the end of instruction list."); } -// instruction ::= name '=' shape opcode operands -bool HloParser::ParseInstruction(HloComputation::Builder* builder) { +// instruction ::= ('ROOT')? name '=' shape opcode operands (extra_attribute)* +bool HloParser::ParseInstruction(HloComputation::Builder* builder, + string* root_name) { string name; Shape shape; HloOpcode opcode; std::vector operands; + bool is_root = EatIfPresent(TokKind::kw_ROOT); if (!ParseName(&name) || !ParseToken(TokKind::kEqual, "expects '=' in instruction") || !ParseShape(&shape) || !ParseOpcode(&opcode)) { return false; } + if (is_root) { + *root_name = name; + } + HloInstruction* instruction; switch (opcode) { case HloOpcode::kParameter: { int64 parameter_number; - return ParseToken(TokKind::kLparen, - "expects '(' before parameter number") && - ParseInt64(¶meter_number) && - ParseToken(TokKind::kRparen, - "expects ')' after parameter number") && - AddInstruction( - name, builder->AddInstruction(HloInstruction::CreateParameter( - parameter_number, shape, name))); + if (!ParseToken(TokKind::kLparen, + "expects '(' before parameter number") || + !ParseInt64(¶meter_number) || + !ParseToken(TokKind::kRparen, "expects ')' after parameter number")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateParameter(parameter_number, shape, name)); + break; } case HloOpcode::kConstant: { std::unique_ptr literal; - return ParseToken(TokKind::kLparen, - "expects '(' before parameter number") && - ParseLiteral(&literal, shape) && - ParseToken(TokKind::kRparen, - "expects ')' after parameter number") && - AddInstruction( - name, builder->AddInstruction( - HloInstruction::CreateConstant(std::move(literal)))); + if (!ParseToken(TokKind::kLparen, + "expects '(' before constant literal") || + !ParseLiteral(&literal, shape) || + !ParseToken(TokKind::kRparen, "expects ')' after constant literal")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateConstant(std::move(literal))); + break; } // Unary ops. case HloOpcode::kAbs: @@ -192,10 +244,12 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder) { case HloOpcode::kSin: case HloOpcode::kSort: case HloOpcode::kTanh: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction(name, - builder->AddInstruction(HloInstruction::CreateUnary( - shape, opcode, operands[0]))); + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateUnary(shape, opcode, operands[0])); + break; } // Binary ops. case HloOpcode::kAdd: @@ -218,46 +272,117 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder) { case HloOpcode::kShiftLeft: case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: { - return ParseOperands(&operands, /*expected_size=*/2) && - AddInstruction( - name, builder->AddInstruction(HloInstruction::CreateBinary( - shape, opcode, operands[0], operands[1]))); + if (!ParseOperands(&operands, /*expected_size=*/2)) { + return false; + } + instruction = builder->AddInstruction(HloInstruction::CreateBinary( + shape, opcode, operands[0], operands[1])); + break; } // Ternary ops. case HloOpcode::kClamp: case HloOpcode::kSelect: { - return ParseOperands(&operands, /*expected_size=*/3) && - AddInstruction( - name, - builder->AddInstruction(HloInstruction::CreateTernary( - shape, opcode, operands[0], operands[1], operands[2]))); + if (!ParseOperands(&operands, /*expected_size=*/3)) { + return false; + } + instruction = builder->AddInstruction(HloInstruction::CreateTernary( + shape, opcode, operands[0], operands[1], operands[2])); + break; } // Other supported ops. case HloOpcode::kConvert: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction( - name, builder->AddInstruction( - HloInstruction::CreateConvert(shape, operands[0]))); + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateConvert(shape, operands[0])); + break; } case HloOpcode::kCrossReplicaSum: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction(name, builder->AddInstruction( - HloInstruction::CreateCrossReplicaSum( - shape, operands[0]))); + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateCrossReplicaSum(shape, operands[0])); + break; } case HloOpcode::kReshape: { - return ParseOperands(&operands, /*expected_size=*/1) && - AddInstruction( - name, builder->AddInstruction( - HloInstruction::CreateReshape(shape, operands[0]))); + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateReshape(shape, operands[0])); + break; + } + case HloOpcode::kTuple: { + if (!ParseOperands(&operands)) { + return false; + } + instruction = + builder->AddInstruction(HloInstruction::CreateTuple(operands)); + break; + } + case HloOpcode::kWhile: { + HloComputation* condition; + HloComputation* body; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseExtraAttribute(&condition, + /*expected_attribute=*/"condition") || + !ParseExtraAttribute(&body, /*expected_attribute=*/"body")) { + return false; + } + instruction = builder->AddInstruction(HloInstruction::CreateWhile( + shape, condition, body, /*init=*/operands[0])); + break; + } + case HloOpcode::kRecv: { + int64 channel_id; + if (!ParseOperands(&operands, /*expected_size=*/0) || + !ParseExtraAttribute(&channel_id, + /*expected_attribute=*/"channel_id")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateRecv(shape, channel_id)); + break; + } + case HloOpcode::kSend: { + int64 channel_id; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseExtraAttribute(&channel_id, + /*expected_attribute=*/"channel_id")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateSend(operands[0], channel_id)); + break; + } + case HloOpcode::kGetTupleElement: { + int64 index; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseExtraAttribute(&index, /*expected_attribute=*/"index")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateGetTupleElement(shape, operands[0], index)); + break; + } + case HloOpcode::kCall: { + HloComputation* to_apply; + if (!ParseOperands(&operands) || + !ParseExtraAttribute(&to_apply, + /*expected_attribute=*/"to_apply")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateCall(shape, operands, to_apply)); + break; } case HloOpcode::kBroadcast: - case HloOpcode::kCall: case HloOpcode::kCustomCall: case HloOpcode::kConcatenate: case HloOpcode::kReducePrecision: case HloOpcode::kConvolution: - case HloOpcode::kGetTupleElement: case HloOpcode::kMap: case HloOpcode::kPad: case HloOpcode::kReduce: @@ -269,22 +394,31 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder) { case HloOpcode::kDynamicSlice: case HloOpcode::kDynamicUpdateSlice: case HloOpcode::kTranspose: - case HloOpcode::kTuple: - case HloOpcode::kWhile: case HloOpcode::kFusion: case HloOpcode::kBatchNormTraining: case HloOpcode::kBatchNormInference: case HloOpcode::kInfeed: case HloOpcode::kOutfeed: case HloOpcode::kBatchNormGrad: - case HloOpcode::kRecv: - case HloOpcode::kSend: case HloOpcode::kUpdate: case HloOpcode::kIndex: case HloOpcode::kTrace: return TokenError(StrCat("parsing not yet implemented for op: ", HloOpcodeString(opcode))); } + // Parse "device=". + if (lexer_.GetKind() == TokKind::kComma) { + int64 device; + if (!ParseExtraAttribute(&device, /*expected_attribute=*/"device")) { + return false; + } + OpDeviceAssignment assignment; + assignment.set_has_device(true); + assignment.set_device(device); + instruction->set_device_assignment(assignment); + } + + return AddInstruction(name, instruction); } bool HloParser::ParseLiteral(std::unique_ptr* literal, @@ -322,8 +456,7 @@ bool HloParser::ParseLiteral(std::unique_ptr* literal, // ::= /*empty*/ // ::= operand (, operand)* // operand ::= shape name -bool HloParser::ParseOperands(std::vector* operands, - const int expected_size) { +bool HloParser::ParseOperands(std::vector* operands) { if (!ParseToken(TokKind::kLparen, "expects '(' at the beginning of operands")) { return false; @@ -345,11 +478,57 @@ bool HloParser::ParseOperands(std::vector* operands, operands->push_back(instruction); } while (EatIfPresent(TokKind::kComma)); } + return ParseToken(TokKind::kRparen, "expects ')' at the end of operands"); +} + +bool HloParser::ParseOperands(std::vector* operands, + const int expected_size) { + if (!ParseOperands(operands)) { + return false; + } if (expected_size != operands->size()) { return TokenError(StrCat("expects ", expected_size, " operands, but has ", operands->size(), " operands")); } - return ParseToken(TokKind::kRparen, "expects ')' at the end of operands"); + return true; +} + +// extra_attribute ::= ',' attribute_name value +template +bool HloParser::ParseExtraAttribute(T* value, + const string& expected_attribute) { + if (!ParseToken(TokKind::kComma, + "expects ',' in front of an extra attribute")) { + return false; + } + string attribute_name; + if (!ParseAttributeName(&attribute_name) && + attribute_name != expected_attribute) { + return TokenError(StrCat("expects attribute name: ", expected_attribute)); + } + if (!ParseAttributeValue(value)) { + return TokenError( + StrCat("expects value for attribute: ", expected_attribute)); + } + return true; +} + +template <> +bool HloParser::ParseAttributeValue(HloComputation** value) { + string name; + if (!ParseName(&name)) { + return TokenError("expects computation name"); + } + *value = tensorflow::gtl::FindPtrOrNull(computation_pool_, name); + if (*value == nullptr) { + return TokenError(StrCat("computation does not exist: ", name)); + } + return true; +} + +template <> +bool HloParser::ParseAttributeValue(int64* value) { + return ParseInt64(value); } // param_list ::= '(' param_list1 ')' @@ -418,6 +597,15 @@ bool HloParser::ParseName(string* result) { return true; } +bool HloParser::ParseAttributeName(string* result) { + if (lexer_.GetKind() != TokKind::kAttributeName) { + return TokenError("expects attribute name"); + } + *result = lexer_.GetStrVal(); + lexer_.Lex(); + return true; +} + bool HloParser::ParseOpcode(HloOpcode* result) { VLOG(1) << "ParseOpcode"; if (lexer_.GetKind() != TokKind::kOpcode) { @@ -488,6 +676,15 @@ bool HloParser::AddInstruction(const string& name, return true; } +bool HloParser::AddComputation(const string& name, + HloComputation* computation) { + auto result = computation_pool_.insert({name, computation}); + if (!result.second) { + return TokenError(StrCat("computation already exists: ", name)); + } + return true; +} + } // namespace StatusOr> Parse(StringPiece str) { diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc index 4ecece3eac..5150e1f96d 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -45,7 +45,7 @@ ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { %x = f32[2,4]{1,0} parameter(1) %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x) %y = f32[2,4]{1,0} parameter(2) - %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y) + ROOT %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y) } )" @@ -56,7 +56,7 @@ ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { R"(HloModule constant_pred_module: ENTRY %constant_pred () -> pred[] { - %constant = pred[] constant(true) + ROOT %constant = pred[] constant(true) } )" @@ -67,7 +67,7 @@ ENTRY %constant_pred () -> pred[] { R"(HloModule constant_s32_module: ENTRY %constant_s32 () -> s32[] { - %constant = s32[] constant(-42) + ROOT %constant = s32[] constant(-42) } )" @@ -77,7 +77,7 @@ ENTRY %constant_s32 () -> s32[] { "ConstantF32", R"(HloModule ConstantF32_module: ENTRY %ConstantF32.v4 () -> f32[] { - %constant = f32[] constant(42) + ROOT %constant = f32[] constant(42) } )" @@ -89,7 +89,7 @@ R"(HloModule add_constants_module: ENTRY %add_constants () -> f32[] { %constant = f32[] constant(3.14) - %add = f32[] add(f32[] %constant, f32[] %constant) + ROOT %add = f32[] add(f32[] %constant, f32[] %constant) } )" @@ -103,7 +103,100 @@ ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f3 %v1 = f32[4]{0} parameter(0) %v2 = f32[4]{0} parameter(1) %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2) - %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) + ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) +} + +)" +}, +// empty tuple +{ +"EmptyTupleCreate", +R"(HloModule EmptyTupleCreate_module: + +ENTRY %EmptyTupleCreate.v1 () -> () { + ROOT %tuple = () tuple() +} + +)" +}, +// tuple +{ +"TupleCreate", +R"(HloModule TupleCreate_module: + +ENTRY %TupleCreate.v4 (v1: f32[], v2: f32[3], v3: f32[2,3]) -> (f32[], f32[3], f32[2,3]) { + %v1 = f32[] parameter(0) + %v2 = f32[3]{0} parameter(1) + %v3 = f32[2,3]{1,0} parameter(2) + ROOT %tuple = (f32[], f32[3]{0}, f32[2,3]{1,0}) tuple(f32[] %v1, f32[3]{0} %v2, f32[2,3]{1,0} %v3) +} + +)" +}, +// int32 result = 0; +// while (result < 5) { result = result + 1; } +{ +"WhileWithScalarS32Result", +R"(HloModule WhileWithScalarS32Result_module: + +%body.v3 (prev.1: s32[]) -> s32[] { + %constant = s32[] constant(1) + %prev.1 = s32[] parameter(0) + ROOT %add = s32[] add(s32[] %constant, s32[] %prev.1) +} + +%condition.v3 (prev.2: s32[]) -> pred[] { + %constant.1 = s32[] constant(5) + %prev.2 = s32[] parameter(0) + ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %prev.2) +} + +ENTRY %WhileWithScalarS32Result.v2 () -> s32[] { + %constant.2 = s32[] constant(0) + ROOT %while = s32[] while(s32[] %constant.2), condition=%condition.v3, body=%body.v3 +} + +)" +}, +// send and recv +{ +"SendRecv", +R"(HloModule TwoSendRecvBothWayRecvFist_module: + +ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> f32[] { + %recv = f32[] recv(), channel_id=15 + ROOT %constant = f32[] constant(2.1) + %send = () send(f32[] %constant), channel_id=16 +} + +)" +}, +// get-tuple-element +{ +"GetTupleElement", +R"(HloModule GetTupleElement_module: + +ENTRY %GetTupleElement.v4 () -> s32[] { + %constant = f32[] constant(1.23) + %constant.1 = s32[] constant(4) + %tuple = (f32[], s32[]) tuple(f32[] %constant, s32[] %constant.1) + ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1 +} + +)" +}, +// call +{ +"Call", +R"(HloModule CallR0F32IdentityScalar_module: + +%Identity.v1 (x: f32[]) -> f32[] { + ROOT %x = f32[] parameter(0) +} + +ENTRY %CallR0F32IdentityScalar.v2 () -> f32[] { + %constant = f32[] constant(42) + ROOT %call = f32[] call(f32[] %constant), to_apply=%Identity.v1 } )" @@ -223,18 +316,6 @@ ENTRY %ConstantWithExp.v4 () -> f32[] { // printed as "300". } -TEST_F(HloParserTest, Tuple) { - const string original = R"(HloModule EmptyTupleCreate_module: - -ENTRY %EmptyTupleCreate.v1 () -> () { - %tuple = () tuple() -} - -)"; - auto result = Parse(original); - EXPECT_NE(tensorflow::Status::OK(), result.status()); -} - } // namespace } // namespace tools } // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h index 1f75e17c7f..1d56ea3478 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_token.h +++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h @@ -41,15 +41,17 @@ enum class TokKind { // Keywords kw_HloModule, kw_ENTRY, + kw_ROOT, kw_true, kw_false, // Typed tokens. - kName, // %foo - kShape, // f32[2,3]{1,0} - kOpcode, // add - kInt, // 42 - kDecimal, // 4.2 + kName, // %foo + kAttributeName, // dimensions= + kShape, // f32[2,3]{1,0} + kOpcode, // add + kInt, // 42 + kDecimal, // 4.2 }; } // namespace tools -- GitLab From 4652341b0a80f93aa06a1b8669f04cb825336af6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 24 Oct 2017 21:10:50 -0700 Subject: [PATCH 340/573] Better hint in eager device placement error message. PiperOrigin-RevId: 173352246 --- tensorflow/c/eager/c_api.cc | 16 ++++++++++++---- tensorflow/python/eager/ops_test.py | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 28ea2edee4..8359de62b7 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -440,11 +440,19 @@ tensorflow::Status ValidateInputTypeAndPlacement( if (expected_device != actual_device) { switch (ctx->policy) { case TFE_DEVICE_PLACEMENT_EXPLICIT: + // TODO(xpan): See if we could bubble python related error up + // to python level. return tensorflow::errors::InvalidArgument( - "cannot compute ", op->name, " as input #", i, - " was expected to be on ", expected_device->name(), - " but is actually on ", actual_device->name(), - " (operation running on ", op_device->name(), ")"); + "Tensors on conflicting devices:" + " cannot compute ", + op->name, " as input #", i, " was expected to be on ", + expected_device->name(), " but is actually on ", + actual_device->name(), " (operation running on ", + op_device->name(), ")", + " Tensors can be copied explicitly using .gpu() or .cpu()," + " or transparently copied by using tfe.enable_eager_execution(" + "tfe.DEVICE_PLACEMENT_SILENT). Copying tensors between devices" + " may slow down your model"); case TFE_DEVICE_PLACEMENT_WARN: LOG(WARNING) << "before computing " << op->name << " input #" << i << " was expected to be on " << expected_device->name() diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 1cd3826755..e34587d5b1 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -249,7 +249,8 @@ class OpsTest(test_util.TensorFlowTestCase): # it should implicitly copy the tensor to host memory? with self.assertRaisesRegexp( errors.InvalidArgumentError, - 'cannot compute Reshape as input #1 was expected to be on'): + 'cannot compute Reshape as input #1 was expected to be on.*' + 'using.*DEVICE_PLACEMENT_SILENT'): reshaped = array_ops.reshape(value, shape.gpu()) def testInvalidInputDataType(self): -- GitLab From ba3fa7f7732bf74341debee3661337d8d836681f Mon Sep 17 00:00:00 2001 From: Kay Zhu Date: Tue, 24 Oct 2017 23:42:11 -0700 Subject: [PATCH 341/573] [XLA] Remove the assumption that the non-CPU backend is the default if more than 1 platforms exist, now that ComputeConstant no longer requires a dedicated CPU backend. PiperOrigin-RevId: 173360476 --- tensorflow/compiler/xla/service/platform_util.cc | 9 --------- tensorflow/compiler/xla/service/platform_util.h | 7 +------ 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc index 4f915a0c2e..3a1818de82 100644 --- a/tensorflow/compiler/xla/service/platform_util.cc +++ b/tensorflow/compiler/xla/service/platform_util.cc @@ -84,15 +84,6 @@ PlatformUtil::GetSupportedPlatforms() { return NotFound("no platforms found"); } else if (platforms.size() == 1) { return platforms[0]; - } else if (platforms.size() == 2) { - // In the service we always link the cpu backend for ComputeConstant. So if - // one of the two platforms is CPU then pick the other (non-cpu) platform as - // the default. - if (platforms[0]->id() == se::host::kHostPlatformId) { - return platforms[1]; - } else if (platforms[1]->id() == se::host::kHostPlatformId) { - return platforms[0]; - } } // Multiple platforms present and we can't pick a reasonable default. diff --git a/tensorflow/compiler/xla/service/platform_util.h b/tensorflow/compiler/xla/service/platform_util.h index fe0281a69a..eac5737030 100644 --- a/tensorflow/compiler/xla/service/platform_util.h +++ b/tensorflow/compiler/xla/service/platform_util.h @@ -36,12 +36,7 @@ class PlatformUtil { // Convenience function which returns the default supported platform. If // exactly one supported platform is present, then this platform is the - // default platform. If exactly two supported platforms are present and one - // platform is CPU (host) then the non-CPU platform is default. This logic is - // used because the XLA service always links in the CPU backend to run - // ComputeConstant, so if exactly one other platform is linked in, we assume - // the intent is to execute on that non-CPU platform. If none of these - // conditions are met the function returns an error. + // default platform. Otherwise returns an error. static StatusOr GetDefaultPlatform(); // Returns a vector of StreamExecutors for the given platform. The vector is -- GitLab From 7828529df0abbbe9bf6c324616857b6e20636a2a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 03:36:25 -0700 Subject: [PATCH 342/573] Internal change PiperOrigin-RevId: 173378236 --- tensorflow/compiler/xla/tests/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 2ea7b9bd8e..4e1be24b61 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -920,6 +920,7 @@ xla_test( name = "reduce_window_test", timeout = "long", srcs = [], + tags = ["optonly"], xla_test_library_deps = [":reduce_window_test_library"], deps = [], ) -- GitLab From 4251fc8e038efd6e18188ee5c1d2dbfa8418e58a Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Wed, 25 Oct 2017 05:25:16 -0700 Subject: [PATCH 343/573] TFE: Adjust formatting of @compatibility(eager) PiperOrigin-RevId: 173385051 --- .../python/ops/resource_variable_ops.py | 7 ++-- tensorflow/python/training/optimizer.py | 38 ++++++++++--------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index ce81a32924..06c5a3bb2a 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -180,7 +180,7 @@ class ResourceVariable(variables.Variable): @compatibility(eager) When Eager Execution is enabled, the default for the `collections` argument - is None, which signifies that this Variable will not be added to any + is `None`, which signifies that this `Variable` will not be added to any collections. @end_compatibility """ @@ -257,8 +257,9 @@ class ResourceVariable(variables.Variable): @compatibility(eager) When Eager Execution is enabled, variables are never added to collections. - It is not implicitly added to the GLOBAL_VARIABLES or TRAINABLE_VARIABLES - collections, and the `collections` argument is ignored. + It is not implicitly added to the `GLOBAL_VARIABLES` or + `TRAINABLE_VARIABLES` collections, and the `collections` argument is + ignored. @end_compatibility """ if initial_value is None: diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index 82fc4edbcd..d6ca52cd1b 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -337,15 +337,16 @@ class Optimizer(object): Raises: ValueError: If some of the variables are not `Variable` objects. - @compatibility(eager): - When eager execution is enabled, `loss` should be a Python function that - takes elements of `var_list` as arguments and computes the value to be - minimized. If `var_list` is None, `loss` should take no arguments. - Minimization (and gradient computation) is done with respect to the - elements of `var_list` if not None, else with respect to any trainable - variables created during the execution of the `loss` function. - `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and - `grad_loss` are ignored when eager execution is enabled. + @compatibility(eager) + When eager execution is enabled, `loss` should be a Python function that + takes elements of `var_list` as arguments and computes the value to be + minimized. If `var_list` is None, `loss` should take no arguments. + Minimization (and gradient computation) is done with respect to the + elements of `var_list` if not None, else with respect to any trainable + variables created during the execution of the `loss` function. + `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and + `grad_loss` are ignored when eager execution is enabled. + @end_compatibility """ grads_and_vars = self.compute_gradients( loss, var_list=var_list, gate_gradients=gate_gradients, @@ -397,15 +398,16 @@ class Optimizer(object): TypeError: If `var_list` contains anything else than `Variable` objects. ValueError: If some arguments are invalid. - @compatibility(eager): - When eager execution is enabled, `loss` should be a Python function that - takes elements of `var_list` as arguments and computes the value to be - minimized. If `var_list` is None, `loss` should take no arguments. - Gradient computation is done with respect to the elements of `var_list` if - not None, else with respect to any trainable variables created during the - execution of the `loss` function. - `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and - `grad_loss` are ignored when eager execution is enabled. + @compatibility(eager) + When eager execution is enabled, `loss` should be a Python function that + takes elements of `var_list` as arguments and computes the value to be + minimized. If `var_list` is None, `loss` should take no arguments. + Gradient computation is done with respect to the elements of `var_list` if + not None, else with respect to any trainable variables created during the + execution of the `loss` function. + `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and + `grad_loss` are ignored when eager execution is enabled. + @end_compatibility """ if context.in_eager_mode(): if grad_loss is not None: -- GitLab From cac07d9c48afcae3ef09c07b8e5cab4eba285b2e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 06:41:31 -0700 Subject: [PATCH 344/573] Small doc cleanup PiperOrigin-RevId: 173390627 --- tensorflow/compiler/xla/service/hlo_constant_folding.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc index b30c7b417f..c05bbeb5c9 100644 --- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc +++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc @@ -63,8 +63,8 @@ StatusOr HloConstantFolding::Run(HloModule* module) { continue; } - // Broadcasts dramatically increase the size of constants with is often - // detrimental to performance and memory capacity so do not fold + // Broadcasts dramatically increase the size of constants, which is often + // detrimental to performance and memory capacity, so do not fold // broadcasts. if (instruction->opcode() == HloOpcode::kBroadcast) { continue; -- GitLab From 3d5d8732508b52e8697b616670dea39beabe8bcb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 07:33:40 -0700 Subject: [PATCH 345/573] Remove unused dependency PiperOrigin-RevId: 173395407 --- tensorflow/compiler/xla/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 0129c51a09..660f419e46 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -340,7 +340,6 @@ cc_library( hdrs = ["array.h"], deps = [ ":types", - ":util", "//tensorflow/core:lib", ], ) -- GitLab From 90278e68c03964fdbb0371357feb7cb1d86bd09b Mon Sep 17 00:00:00 2001 From: Taehoon Lee Date: Wed, 25 Oct 2017 23:57:36 +0900 Subject: [PATCH 346/573] Fix typos --- .../boosted_trees/python/training/functions/gbdt_batch.py | 2 +- tensorflow/examples/get_started/regression/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py index f8f4b43a07..5a917ca428 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py @@ -344,7 +344,7 @@ class GradientBoostedDecisionTreeModel(object): learner_config.num_classes == 2) def _predict_and_return_dict(self, ensemble_handle, ensemble_stamp, mode): - """Runs prediciton and returns a dictionary of the prediction results. + """Runs prediction and returns a dictionary of the prediction results. Args: ensemble_handle: ensemble resource handle. diff --git a/tensorflow/examples/get_started/regression/test.py b/tensorflow/examples/get_started/regression/test.py index 652b44f543..0b1477ad96 100644 --- a/tensorflow/examples/get_started/regression/test.py +++ b/tensorflow/examples/get_started/regression/test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""A simple smoke test that runs these examples for 1 training iteraton.""" +"""A simple smoke test that runs these examples for 1 training iteration.""" from __future__ import absolute_import from __future__ import division -- GitLab From 2eeb6df0bd7c329163a6a25dd111a25a7b9ad16f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 08:34:05 -0700 Subject: [PATCH 347/573] Fix SIGSEGV in GraphRunner::Run when called with a function library for a non-CPU device. PiperOrigin-RevId: 173401446 --- tensorflow/core/common_runtime/graph_runner.cc | 11 +++++++++++ tensorflow/python/framework/function_test.py | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/common_runtime/graph_runner.cc b/tensorflow/core/common_runtime/graph_runner.cc index d0f9e6ed18..a21304f7ef 100644 --- a/tensorflow/core/common_runtime/graph_runner.cc +++ b/tensorflow/core/common_runtime/graph_runner.cc @@ -109,6 +109,17 @@ Status GraphRunner::Run(Graph* graph, FunctionLibraryRuntime* function_library, return errors::NotFound("Cannot find a device for GraphRunner."); } + if (function_library && function_library->device() && + function_library->device()->device_type() != cpu_device_->device_type()) { + // We are running on a CPU but the function library is for a non-CPU device, + // so just ignore the function_library. + // TODO(matthewmurray) Can we create a new FunctionLibraryRuntime that is + // identical to function_library except that it uses CPU? + VLOG(1) << "Cannot run on CPU device with a function library for a " + << function_library->device()->device_type() << " device."; + function_library = nullptr; + } + // TODO(vrv): Instead of copying the entire graph, consider modifying // the existing graph, and then removing those removed edges. // prior to returning. diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index fea2129922..fbc1045b5b 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -309,8 +309,7 @@ class FunctionTest(test.TestCase): self.assertAllClose(y.eval(), 6.) self.assertAllClose(dx.eval(), 2.) - def testZNoDepOnY(self): - + def _testZNoDepOnY(self, use_const_grad_ys): @function.Defun(dtypes.float32, dtypes.float32) def Foo(x, y): # pylint: disable=unused-argument return x * 2 @@ -320,12 +319,22 @@ class FunctionTest(test.TestCase): x = constant_op.constant(1.0) y = constant_op.constant(2.0) z = Foo(x, y) - dx, dy = gradients_impl.gradients([z], [x, y]) + if use_const_grad_ys: + dx, dy = gradients_impl.gradients([z], [x, y], grad_ys=[1.0]) + else: + dx, dy = gradients_impl.gradients([z], [x, y]) with session.Session() as sess: dx_val, dy_val = sess.run([dx, dy]) self.assertEqual([2.0], dx_val) self.assertEqual([0.0], dy_val) + def testZNoDepOnY(self): + self._testZNoDepOnY(False) + + def testZNoDepOnYConstGradYs(self): + # Tests for constant folding of grad_ys + self._testZNoDepOnY(True) + def testDefineFunctionNoArgs(self): @function.Defun(func_name="AConstant") -- GitLab From 8357c3164689ed2ce5df4df69ef7439b3b2fce82 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 08:56:23 -0700 Subject: [PATCH 348/573] First part of the refactoring allowing sparse multivalent feature columns. This change extends the split proto to allow feature ids within the feature columns. PiperOrigin-RevId: 173403860 --- .../estimator_batch/custom_export_strategy.py | 4 + tensorflow/contrib/boosted_trees/lib/BUILD | 12 ++ .../boosted_trees/lib/trees/decision_tree.cc | 25 ++- .../lib/trees/decision_tree_test.cc | 145 ++++++++++++------ .../boosted_trees/lib/utils/batch_features.cc | 4 - .../lib/utils/batch_features_test.cc | 13 -- .../contrib/boosted_trees/lib/utils/example.h | 56 ++++++- .../boosted_trees/lib/utils/example_test.cc | 81 ++++++++++ .../lib/utils/examples_iterable.cc | 2 - .../lib/utils/examples_iterable.h | 29 +++- .../lib/utils/examples_iterable_test.cc | 112 +++++++++----- .../lib/utils/sparse_column_iterable.h | 2 + .../lib/utils/sparse_column_iterable_test.cc | 10 +- .../boosted_trees/proto/tree_config.proto | 3 + 14 files changed, 376 insertions(+), 122 deletions(-) create mode 100644 tensorflow/contrib/boosted_trees/lib/utils/example_test.cc diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index a800c3ddc7..ef8dee91b6 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -149,6 +149,8 @@ def convert_to_universal_format(dtec, sorted_feature_names, split = gtflow_node.sparse_float_binary_split_default_left.split node.default_direction = ( generic_tree_model_pb2.BinaryNode.LEFT) + # TODO(nponomareva): adjust this id assignement when we allow multi- + # column sparse tensors. feature_id = split.feature_column + num_dense inequality_test = node.inequality_left_child_test inequality_test.feature_id.id.value = sorted_feature_names[feature_id] @@ -159,6 +161,8 @@ def convert_to_universal_format(dtec, sorted_feature_names, split = gtflow_node.sparse_float_binary_split_default_right.split node.default_direction = ( generic_tree_model_pb2.BinaryNode.RIGHT) + # TODO(nponomareva): adjust this id assignement when we allow multi- + # column sparse tensors. feature_id = split.feature_column + num_dense inequality_test = node.inequality_left_child_test inequality_test.feature_id.id.value = sorted_feature_names[feature_id] diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD index 70aa0284a6..107ff0d295 100644 --- a/tensorflow/contrib/boosted_trees/lib/BUILD +++ b/tensorflow/contrib/boosted_trees/lib/BUILD @@ -81,6 +81,18 @@ tf_cc_test( ], ) +tf_cc_test( + name = "example_test", + size = "small", + srcs = ["utils/example_test.cc"], + deps = [ + ":utils", + "//tensorflow/core:tensor_testutil", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + tf_cc_test( name = "batch_features_test", size = "small", diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc index bd70586393..f8750e7191 100644 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc +++ b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc @@ -50,10 +50,15 @@ int DecisionTree::Traverse(const DecisionTreeConfig& config, current_node.sparse_float_binary_split_default_left().split(); auto sparse_feature = example.sparse_float_features[split.feature_column()]; - node_id = !sparse_feature.has_value() || - sparse_feature.get_value() <= split.threshold() - ? split.left_id() - : split.right_id(); + // Feature id for the split when multivalent sparse float column, or 0 + // by default. + const int32 feature_id = split.feature_id(); + + node_id = + !sparse_feature[feature_id].has_value() || + sparse_feature[feature_id].get_value() <= split.threshold() + ? split.left_id() + : split.right_id(); break; } case TreeNode::kSparseFloatBinarySplitDefaultRight: { @@ -61,10 +66,14 @@ int DecisionTree::Traverse(const DecisionTreeConfig& config, current_node.sparse_float_binary_split_default_right().split(); auto sparse_feature = example.sparse_float_features[split.feature_column()]; - node_id = sparse_feature.has_value() && - sparse_feature.get_value() <= split.threshold() - ? split.left_id() - : split.right_id(); + // Feature id for the split when multivalent sparse float column, or 0 + // by default. + const int32 feature_id = split.feature_id(); + node_id = + sparse_feature[feature_id].has_value() && + sparse_feature[feature_id].get_value() <= split.threshold() + ? split.left_id() + : split.right_id(); break; } case TreeNode::kCategoricalIdBinarySplit: { diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc index c55d09807e..93924d429c 100644 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc @@ -27,13 +27,14 @@ class DecisionTreeTest : public ::testing::Test { protected: DecisionTreeTest() : batch_features_(2) { // Create a batch of two examples having one dense float, two sparse float - // and one sparse int features. + // and one sparse int features, and one sparse multi-column float feature + // (SparseFM). // The first example is missing the second sparse feature column and the // second example is missing the first sparse feature column. // This looks like the following: - // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | - // 0 | 7 | -3 | | 3 | - // 1 | -2 | | 4 | | + // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseFM (3 cols) + // 0 | 7 | -3 | | 3 | 3.0 | | 1.0 + // 1 | -2 | | 4 | | 1.5 |3.5| auto dense_float_matrix = test::AsTensor({7.0f, -2.0f}, {2, 1}); auto sparse_float_indices1 = test::AsTensor({0, 0}, {1, 2}); auto sparse_float_values1 = test::AsTensor({-3.0f}); @@ -44,11 +45,21 @@ class DecisionTreeTest : public ::testing::Test { auto sparse_int_indices1 = test::AsTensor({0, 0}, {1, 2}); auto sparse_int_values1 = test::AsTensor({3}); auto sparse_int_shape1 = test::AsTensor({2, 1}); + + // Multivalent sparse feature. + auto multi_sparse_float_indices = + test::AsTensor({0, 0, 0, 2, 1, 0, 1, 1}, {4, 2}); + auto multi_sparse_float_values = + test::AsTensor({3.0f, 1.0f, 1.5f, 3.5f}); + auto multi_sparse_float_shape = test::AsTensor({2, 3}); + TF_EXPECT_OK(batch_features_.Initialize( - {dense_float_matrix}, {sparse_float_indices1, sparse_float_indices2}, - {sparse_float_values1, sparse_float_values2}, - {sparse_float_shape1, sparse_float_shape2}, {sparse_int_indices1}, - {sparse_int_values1}, {sparse_int_shape1})); + {dense_float_matrix}, + {sparse_float_indices1, sparse_float_indices2, + multi_sparse_float_indices}, + {sparse_float_values1, sparse_float_values2, multi_sparse_float_values}, + {sparse_float_shape1, sparse_float_shape2, multi_sparse_float_shape}, + {sparse_int_indices1}, {sparse_int_values1}, {sparse_int_shape1})); } template @@ -121,44 +132,90 @@ TEST_F(DecisionTreeTest, TraverseDenseBinarySplit) { } TEST_F(DecisionTreeTest, TraverseSparseBinarySplit) { - // Test first sparse feature which is missing for the second example. - DecisionTreeConfig tree_config1; - auto* split_node1 = tree_config1.add_nodes() - ->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - split_node1->set_feature_column(0); - split_node1->set_threshold(-20.0f); - split_node1->set_left_id(1); - split_node1->set_right_id(2); - tree_config1.add_nodes()->mutable_leaf(); - tree_config1.add_nodes()->mutable_leaf(); auto example_iterable = batch_features_.examples_iterable(0, 2); - - // Expect right child to be picked as !(-3 <= -20). - auto example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config1, 0, *example_it)); - - // Expect left child to be picked as default direction. - EXPECT_EQ(1, DecisionTree::Traverse(tree_config1, 0, *++example_it)); - + // Split on SparseF1. + // Test first sparse feature which is missing for the second example. + { + DecisionTreeConfig tree_config; + auto* split_node = tree_config.add_nodes() + ->mutable_sparse_float_binary_split_default_left() + ->mutable_split(); + split_node->set_feature_column(0); + split_node->set_threshold(-20.0f); + split_node->set_left_id(1); + split_node->set_right_id(2); + tree_config.add_nodes()->mutable_leaf(); + tree_config.add_nodes()->mutable_leaf(); + + // Expect right child to be picked as !(-3 <= -20). + auto example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + + // Expect left child to be picked as default direction. + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + } + // Split on SparseF2. // Test second sparse feature which is missing for the first example. - DecisionTreeConfig tree_config2; - auto* split_node2 = tree_config2.add_nodes() - ->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - split_node2->set_feature_column(1); - split_node2->set_threshold(4.0f); - split_node2->set_left_id(1); - split_node2->set_right_id(2); - tree_config2.add_nodes()->mutable_leaf(); - tree_config2.add_nodes()->mutable_leaf(); - - // Expect right child to be picked as default direction. - example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config2, 0, *example_it)); - - // Expect left child to be picked as (4 <= 4). - EXPECT_EQ(1, DecisionTree::Traverse(tree_config2, 0, *++example_it)); + { + DecisionTreeConfig tree_config; + auto* split_node = tree_config.add_nodes() + ->mutable_sparse_float_binary_split_default_right() + ->mutable_split(); + split_node->set_feature_column(1); + split_node->set_threshold(4.0f); + split_node->set_left_id(1); + split_node->set_right_id(2); + tree_config.add_nodes()->mutable_leaf(); + tree_config.add_nodes()->mutable_leaf(); + + // Expect right child to be picked as default direction. + auto example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + + // Expect left child to be picked as (4 <= 4). + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + } + // Split on SparseFM. + // Test second sparse feature which is missing for the first example. + { + DecisionTreeConfig tree_config; + auto* split_node = tree_config.add_nodes() + ->mutable_sparse_float_binary_split_default_right() + ->mutable_split(); + split_node->set_feature_column(2); + + split_node->set_left_id(1); + split_node->set_right_id(2); + tree_config.add_nodes()->mutable_leaf(); + tree_config.add_nodes()->mutable_leaf(); + + // Split on first column + split_node->set_feature_id(0); + split_node->set_threshold(2.0f); + + // Both instances have this feature value. + auto example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + + // Split on second column + split_node->set_feature_id(1); + split_node->set_threshold(5.0f); + + // First instance does not have it (default right), second does have it. + example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + + // Split on third column + split_node->set_feature_id(2); + split_node->set_threshold(3.0f); + example_it = example_iterable.begin(); + + // First instance has it, second does not (default right). + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *example_it)); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *++example_it)); + } } TEST_F(DecisionTreeTest, TraverseCategoricalIdBinarySplit) { diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc index 12b377dda7..cf4f9a097a 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc @@ -94,10 +94,6 @@ Status BatchFeatures::Initialize( shape_flat(0) == batch_size_, errors::InvalidArgument( "Sparse float feature shape incompatible with batch size.")); - TF_CHECK_AND_RETURN_IF_ERROR( - shape_flat(1) <= 1, - errors::InvalidArgument( - "Sparse float features may not be multi-valent.")); auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)}); auto order_dims = sparse::SparseTensor::VarDimArray({0, 1}); sparse_float_feature_columns_.emplace_back(sparse_float_feature_indices, diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc index 7f523d527a..9de3e32b09 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc @@ -129,19 +129,6 @@ TEST_F(BatchFeaturesTest, SparseFloatFeatures_IncompatibleShape) { {sparse_float_feature_shape}, {}, {}, {})); } -TEST_F(BatchFeaturesTest, SparseFloatFeatures_Multivalent) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}); - auto sparse_float_feature_shape = AsTensor({2, 2}); - auto expected_error = - InvalidArgument("Sparse float features may not be multi-valent."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongShapeIndices) { BatchFeatures batch_features(2); auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}); diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example.h b/tensorflow/contrib/boosted_trees/lib/utils/example.h index 4681eb06aa..9514416660 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/example.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/example.h @@ -16,6 +16,8 @@ #ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ #define THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ +#include +#include #include #include #include "tensorflow/contrib/boosted_trees/lib/utils/optional_value.h" @@ -24,6 +26,56 @@ namespace tensorflow { namespace boosted_trees { namespace utils { +// A matrix that given feature column id and feature value id will return +// either a value or an optional. First index indicates feature column, second +// index - the index of the value within this column - for single valued, it +// will be 0. +// Allows double-subscript access [][]. +template +class SparseMatrix { + typedef std::vector> SparseMap; + + class Proxy { + public: + Proxy(const int32 feature_column_idx, const SparseMap& values) + : feature_column_idx_(feature_column_idx), values_(values) {} + + OptionalValue operator[](int feature_idx) const { + auto value_iter = std::find_if( + values_.begin(), values_.end(), + [this, &feature_idx](const std::tuple& element) { + return std::get<0>(element) == feature_column_idx_ && + std::get<1>(element) == feature_idx; + }); + + if (value_iter == values_.end()) { + return OptionalValue(); + } + // There is this feature column and feature id. + return OptionalValue(std::get<2>(*value_iter)); + } + + private: + int32 feature_column_idx_; + const SparseMap& values_; + }; + + public: + void addElement(const int32 feature_column_idx, const int32 feature_idx, + const T value) { + values_.emplace_back(feature_column_idx, feature_idx, value); + } + + void clear() { values_.clear(); } + + Proxy operator[](int feature_column_idx) const { + return Proxy(feature_column_idx, values_); + } + + private: + SparseMap values_; +}; + // Holds data for one example and enables lookup by feature column. struct Example { // Default constructor creates an empty example. @@ -35,7 +87,9 @@ struct Example { // Dense and sparse float features indexed by feature column. // TODO(salehay): figure out a design to support multivalent float features. std::vector dense_float_features; - std::vector> sparse_float_features; + // Sparse float features are allowed to be multivalent and thus can be + // represented as a sparse matrix. + SparseMatrix sparse_float_features; // Sparse integer features indexed by feature column. // Note that all integer features are assumed to be categorical, i.e. will diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc new file mode 100644 index 0000000000..f78fd25022 --- /dev/null +++ b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc @@ -0,0 +1,81 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= +#include "tensorflow/contrib/boosted_trees/lib/utils/example.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace boosted_trees { +namespace utils { +namespace { + +class ExampleTest : public ::testing::Test {}; + +TEST_F(ExampleTest, TestSparseMatrix) { + // Create the following matrix: + // row id | | 0.4 | 0.3 + // 0 | 1 | | 2 + // 1 | 3 | 1 | 5 + // 2 | | | -4 + // 3 | | | + SparseMatrix matrix; + matrix.addElement(0, 1, 0.4f); + matrix.addElement(0, 2, 0.3f); + matrix.addElement(1, 0, 1.f); + matrix.addElement(1, 2, 2.f); + matrix.addElement(2, 0, 3.f); + matrix.addElement(2, 1, 1.f); + matrix.addElement(2, 2, 5.f); + matrix.addElement(3, 2, -4.f); + + // Row 0. + EXPECT_FALSE(matrix[0][0].has_value()); + EXPECT_TRUE(matrix[0][1].has_value()); + EXPECT_EQ(0.4f, matrix[0][1].get_value()); + EXPECT_TRUE(matrix[0][2].has_value()); + EXPECT_EQ(0.3f, matrix[0][2].get_value()); + + // Row 1. + EXPECT_TRUE(matrix[1][0].has_value()); + EXPECT_EQ(1.f, matrix[1][0].get_value()); + EXPECT_FALSE(matrix[1][1].has_value()); + EXPECT_TRUE(matrix[1][2].has_value()); + EXPECT_EQ(2.f, matrix[1][2].get_value()); + + // Row 2. + EXPECT_TRUE(matrix[2][0].has_value()); + EXPECT_EQ(3.f, matrix[2][0].get_value()); + EXPECT_TRUE(matrix[2][1].has_value()); + EXPECT_EQ(1.f, matrix[2][1].get_value()); + EXPECT_TRUE(matrix[2][2].has_value()); + EXPECT_EQ(5.f, matrix[2][2].get_value()); + + // Row 3. + EXPECT_FALSE(matrix[3][0].has_value()); + EXPECT_FALSE(matrix[3][1].has_value()); + EXPECT_TRUE(matrix[3][2].has_value()); + EXPECT_EQ(-4.f, matrix[3][2].get_value()); + + // Row 4. + EXPECT_FALSE(matrix[4][0].has_value()); + EXPECT_FALSE(matrix[4][1].has_value()); + EXPECT_FALSE(matrix[4][2].has_value()); +} + +} // namespace +} // namespace utils +} // namespace boosted_trees +} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc index c73dc8e15d..3b287b1dcf 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc @@ -73,8 +73,6 @@ Iterator::Iterator(ExamplesIterable* iter, int64 example_idx) // Pre-size example features. example_.dense_float_features.resize( iter_->dense_float_column_values_.size()); - example_.sparse_float_features.resize( - iter_->sparse_float_column_values_.size()); example_.sparse_int_features.resize(iter_->sparse_int_column_values_.size()); } diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h index 67efb82a22..72b7486872 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h @@ -87,19 +87,34 @@ class ExamplesIterable { // Get sparse float values per column. auto& sparse_float_features = example_.sparse_float_features; + sparse_float_features.clear(); + // Iterate through each sparse float feature column. for (size_t sparse_float_idx = 0; - sparse_float_idx < sparse_float_features.size(); + sparse_float_idx < iter_->sparse_float_column_iterables_.size(); ++sparse_float_idx) { + // Get range for values tensor. const auto& row_range = (*sparse_float_column_iterators_[sparse_float_idx]); DCHECK_EQ(example_idx_, row_range.example_idx); + // If the example has this feature column. if (row_range.start < row_range.end) { - DCHECK_EQ(1, row_range.end - row_range.start); - sparse_float_features[sparse_float_idx] = OptionalValue( - iter_->sparse_float_column_values_[sparse_float_idx]( - row_range.start)); - } else { - sparse_float_features[sparse_float_idx] = OptionalValue(); + // Retrieve original indices tensor. + const TTypes::ConstMatrix& indices = + iter_->sparse_float_column_iterables_[sparse_float_idx] + .sparse_indices(); + + // For each value. + for (int64 row_idx = row_range.start; row_idx < row_range.end; + ++row_idx) { + // Get the feature id for the feature column and the value. + const int32 feature_id = indices(row_idx, 1); + DCHECK_EQ(example_idx_, indices(row_idx, 0)); + + // Save the value to our sparse matrix. + sparse_float_features.addElement( + sparse_float_idx, feature_id, + iter_->sparse_float_column_values_[sparse_float_idx](row_idx)); + } } } diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc index d93bcc8aa6..05c166edc6 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc @@ -26,17 +26,17 @@ class ExamplesIterableTest : public ::testing::Test {}; TEST_F(ExamplesIterableTest, Iterate) { // Create a batch of 8 examples having one dense float, two sparse float and - // two sparse int features. + // two sparse int features. Second sparse float feature is multivalent. // The data looks like the following: // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseI2 | - // 0 | 7 | -3 | | 1, 8 | | - // 1 | -2 | | 4 | 0 | 7 | - // 2 | 8 | 0 | | | 13 | - // 3 | 1 | 5 | 7 | 2, 0 | 4 | - // 4 | 0 | 0 | | | 0 | - // 5 | -4 | | 9 | | | - // 6 | 7 | | | | | - // 7 | -2 | | -4 | 5 | | + // 0 | 7 | -3 | | 1 | 1, 8 | | + // 1 | -2 | | 4 | | 0 | 7 | + // 2 | 8 | 0 | | 3 | | 13 | + // 3 | 1 | 5 | 7 | | 2, 0 | 4 | + // 4 | 0 | 0 | | 4.3 | | 0 | + // 5 | -4 | | 9 | 0.8 | | | + // 6 | 7 | | | | | | + // 7 | -2 | | -4 | | 5 | | auto dense_float_tensor = test::AsTensor( {7.0f, -2.0f, 8.0f, 1.0f, 0.0f, -4.0f, 7.0f, -2.0f}, {8, 1}); auto sparse_float_indices1 = @@ -45,10 +45,11 @@ TEST_F(ExamplesIterableTest, Iterate) { auto sparse_float_shape1 = TensorShape({8, 1}); sparse::SparseTensor sparse_float_tensor1( sparse_float_indices1, sparse_float_values1, sparse_float_shape1); - auto sparse_float_indices2 = - test::AsTensor({1, 0, 3, 0, 5, 0, 7, 0}, {4, 2}); - auto sparse_float_values2 = test::AsTensor({4.0f, 7.0f, 9.0f, -4.0f}); - auto sparse_float_shape2 = TensorShape({8, 1}); + auto sparse_float_indices2 = test::AsTensor( + {0, 1, 1, 0, 2, 1, 3, 0, 4, 1, 5, 0, 5, 1, 7, 0}, {8, 2}); + auto sparse_float_values2 = + test::AsTensor({1.f, 4.0f, 3.f, 7.0f, 4.3f, 9.0f, 0.8f, -4.0f}); + auto sparse_float_shape2 = TensorShape({8, 2}); sparse::SparseTensor sparse_float_tensor2( sparse_float_indices2, sparse_float_values2, sparse_float_shape2); auto sparse_int_indices1 = @@ -67,15 +68,19 @@ TEST_F(ExamplesIterableTest, Iterate) { auto validate_example_features = [](int64 example_idx, const Example& example) { EXPECT_EQ(1, example.dense_float_features.size()); - EXPECT_EQ(2, example.sparse_float_features.size()); switch (example_idx) { case 0: { EXPECT_EQ(0, example.example_idx); EXPECT_EQ(7.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(-3.0f, example.sparse_float_features[0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(-3.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2 - multivalent. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(1.0f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(2, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(1)); EXPECT_EQ(1, example.sparse_int_features[0].count(8)); @@ -84,9 +89,13 @@ TEST_F(ExamplesIterableTest, Iterate) { case 1: { EXPECT_EQ(1, example.example_idx); EXPECT_EQ(-2.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(4.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(4.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(1, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(0)); EXPECT_EQ(1, example.sparse_int_features[1].size()); @@ -95,9 +104,14 @@ TEST_F(ExamplesIterableTest, Iterate) { case 2: { EXPECT_EQ(2, example.example_idx); EXPECT_EQ(8.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(0.0f, example.sparse_float_features[0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(3.f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[1].size()); EXPECT_EQ(1, example.sparse_int_features[1].count(13)); @@ -105,10 +119,14 @@ TEST_F(ExamplesIterableTest, Iterate) { case 3: { EXPECT_EQ(3, example.example_idx); EXPECT_EQ(1.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(5.0f, example.sparse_float_features[0].get_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(7.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(5.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(7.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(2, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(2)); EXPECT_EQ(1, example.sparse_int_features[0].count(0)); @@ -118,9 +136,14 @@ TEST_F(ExamplesIterableTest, Iterate) { case 4: { EXPECT_EQ(4, example.example_idx); EXPECT_EQ(0.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(0.0f, example.sparse_float_features[0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(4.3f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[1].size()); EXPECT_EQ(1, example.sparse_int_features[1].count(0)); @@ -128,24 +151,37 @@ TEST_F(ExamplesIterableTest, Iterate) { case 5: { EXPECT_EQ(5, example.example_idx); EXPECT_EQ(-4.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(9.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(9.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(0.8f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); } break; case 6: { EXPECT_EQ(6, example.example_idx); EXPECT_EQ(7.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); } break; case 7: { EXPECT_EQ(7, example.example_idx); EXPECT_EQ(-2.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(-4.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(-4.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(1, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(5)); } break; diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h index 78a5752730..9664c9d1c6 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h @@ -112,6 +112,8 @@ class SparseColumnIterable { int64 example_start() const { return example_start_; } int64 example_end() const { return example_end_; } + const TTypes::ConstMatrix& sparse_indices() const { return ix_; } + private: // Sparse indices matrix. TTypes::ConstMatrix ix_; diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc index 7792bd8c66..0138aae3db 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc @@ -34,19 +34,19 @@ TEST_F(SparseColumnIterableTest, Empty) { } TEST_F(SparseColumnIterableTest, Iterate) { - // 8 examples having 7 sparse features with the third multi-valent. + // 8 examples having 7 sparse features with the 3rd and 7th multi-valent. // This can be visualized like the following: // Instance | Sparse | - // 0 | x | + // 0 | x | // 1 | | // 2 | | // 3 | xxx | - // 4 | x | + // 4 | x | // 5 | | // 6 | | - // 7 | xx | + // 7 | x x | const auto indices = - AsTensor({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 1}, {7, 2}); + AsTensor({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 2}, {7, 2}); auto validate_example_range = [](const ExampleRowRange& range) { switch (range.example_idx) { diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto index 2e9d45efd7..f14abf45a5 100644 --- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto +++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto @@ -53,6 +53,9 @@ message DenseFloatBinarySplit { // Float feature column and split threshold describing // the rule feature <= threshold. int32 feature_column = 1; + // If feature column is multivalent, this holds the index of the feature for + // the split. Defaults to 0. + int32 feature_id = 5; float threshold = 2; // Node children indexing into a contiguous -- GitLab From d17db4b011d3c04cf7ff8caf4578032b4c0fc622 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 09:03:21 -0700 Subject: [PATCH 349/573] Split Evaluator.evaluate_on_dataset() into two methods to separate graph construction from running in sessions. Otherwise it is very hard to avoid adding to the graph every time you do an eval. Also: * Stop making init_variables() conditional on whether the variables have already been initialized. * Use new feature of group() to accept lists. PiperOrigin-RevId: 173404673 --- tensorflow/contrib/eager/python/evaluator.py | 87 ++++++++++++++++--- .../contrib/eager/python/evaluator_test.py | 3 +- .../contrib/eager/python/metrics_impl.py | 16 +--- .../contrib/eager/python/metrics_test.py | 4 +- 4 files changed, 79 insertions(+), 31 deletions(-) diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py index 67f545e838..633c747e5e 100644 --- a/tensorflow/contrib/eager/python/evaluator.py +++ b/tensorflow/contrib/eager/python/evaluator.py @@ -81,10 +81,18 @@ class Evaluator(object): Returns: An op. + + Raises: + RuntimeError: if eager execution is enabled. + + @compatibility(eager) + Only for graph execution. + @end_compatibility """ - assert context.in_graph_mode() - return control_flow_ops.group( - *[m.init_variables() for _, m in self.metrics]) + if context.in_eager_mode(): + raise RuntimeError("Evaluator.init_variables() not needed when " + "eager execution is enabled.") + return control_flow_ops.group([m.init_variables() for _, m in self.metrics]) def all_metric_results(self): # TODO(josh11b): Add optional summary_writer. """Returns dict mapping metric name -> value.""" @@ -97,28 +105,79 @@ class Evaluator(object): return results def evaluate_on_dataset(self, dataset, *args, **kwargs): - """Convenience method for performing an eval on a Dataset.""" + """Convenience method for performing an eval on a Dataset. + + Args: + dataset: Dataset object with the input data to evaluate on. + *args: + **kwargs: Optional additional arguments to __call__(). + + Returns: + @compatibility(eager) + When eager execution is enabled, this returns the result of performing + an evaluation as a dictionary. With graph execution, this returns a tuple + (init_op, call_op, results_op) which may be executed using this code: + ```python + sess.run(init_op) + try: + while True: + sess.run(call_op) + except tf.errors.OutOfRangeError: + pass + return sess.run(results_op) # A dictionary + + # equivalently: + return evaluator.run_evaluation(init_op, call_op, results_op, sess=sess) + ``` + @end_compatibility + """ # TODO(josh11b): Add optional summary_writer. if context.in_graph_mode(): - # TODO(josh11b): Return an dict of tensors to pass to session.run() - # instead of running using the default session here. - sess = ops.get_default_session() call_op = self.__call__(dataset.make_one_shot_iterator().get_next(), *args, **kwargs) init_op = self.init_variables() results_op = self.all_metric_results() - sess.run(init_op) - try: - while True: - sess.run(call_op) - except errors_impl.OutOfRangeError: - pass - return sess.run(results_op) + return (init_op, call_op, results_op) # Eager case for example in datasets.Iterator(dataset): self.__call__(example, *args, **kwargs) return self.all_metric_results() + @staticmethod + def run_evaluation(init_op, call_op, results_op, sess=None): + """Convenience method for running the ops returned by evaluate_on_dataset. + + Args: + init_op: An op that initializes/resets evaluation state. + call_op: An op that updates evaluation state on a mini-batch of examples. + Must generate an tf.errors.OutOfRangeError when done. + results_op: A dictionary of tensors that compute the final evaluation + results from the evaulation state. + sess: The Session to run the evaluation in. Defaults to the default + Session. + + Returns: + A dictionary of values, parallel to results_op. + + Raises: + RuntimeError: if eager execution is enabled. + + @compatibility(eager) + Only for graph execution. + @end_compatibility + """ + if context.in_eager_mode(): + raise RuntimeError("Evaluator.run_evaluation() not supported when " + "eager execution is enabled.") + sess = sess or ops.get_default_session() + sess.run(init_op) + try: + while True: + sess.run(call_op) + except errors_impl.OutOfRangeError: + pass + return sess.run(results_op) + # ---- To be implemented by descendants --- def call(self, eval_data): """Update metrics using the output of self.model. diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py index 71e9fa40a8..4652a69081 100644 --- a/tensorflow/contrib/eager/python/evaluator_test.py +++ b/tensorflow/contrib/eager/python/evaluator_test.py @@ -100,7 +100,8 @@ class EvaluatorTest(test.TestCase): with context.graph_mode(), self.test_session(): e = SimpleEvaluator(IdentityModel()) ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) - results = e.evaluate_on_dataset(ds) + init_op, call_op, results_op = e.evaluate_on_dataset(ds) + results = e.run_evaluation(init_op, call_op, results_op) self.assertEqual(set(["mean"]), set(results.keys())) self.assertEqual(6.0, results["mean"]) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 77a84e006e..6af0d65e08 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -23,30 +23,18 @@ import re from tensorflow.contrib.summary import summary_ops from tensorflow.python.eager import context from tensorflow.python.eager import function -from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope _to_replace = re.compile("[^A-Za-z0-9.]") -def _init_var(v): - def do_init(v): - with ops.control_dependencies([v.assign(v.initial_value)]): - return constant_op.constant(True) - return control_flow_ops.cond( - resource_variable_ops.var_is_initialized_op(v._handle), # pylint: disable=protected-access - lambda: constant_op.constant(False), - lambda: do_init(v)) - - class Metric(object): """A metric holds state for aggregating statistics over an evaluation run. @@ -121,7 +109,7 @@ class Metric(object): return self._vars def init_variables(self): - """Return an op for initializing this Metric's uninitialized variables. + """Return an op for initializing this Metric's variables. Only for graph execution. Should be called after variables are created in the first execution of __call__(). @@ -130,7 +118,7 @@ class Metric(object): An op to run. """ assert context.in_graph_mode() - return control_flow_ops.group(*[_init_var(v) for v in self._vars]) + return control_flow_ops.group([v.initializer for v in self._vars]) # ---- To be implemented by descendants --- def build(self, *args, **kwargs): diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index fce6be1761..3ecbaeae69 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -141,10 +141,10 @@ class MetricsTest(test.TestCase): sess.run(accumulate, feed_dict={p: 1000}) sess.run(accumulate, feed_dict={p: [10000, 100000]}) self.assertAllEqual(m.result().eval(), 111111.0/6) - # Second init is ignored, since the variables are already initialized. + # Second init resets all the variables. init_op.run() sess.run(accumulate, feed_dict={p: 7}) - self.assertAllEqual(m.result().eval(), 111118.0/7) + self.assertAllEqual(m.result().eval(), 7) def testTwoMeansGraph(self): # Verify two metrics with the same class and name don't -- GitLab From 4f86cf60254126d28f5f653d810de0a2cf1473c8 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Wed, 25 Oct 2017 09:33:42 -0700 Subject: [PATCH 350/573] [TX2XLA] Add function name when reporting signature check failure. PiperOrigin-RevId: 173408083 --- tensorflow/compiler/tf2xla/xla_compiler.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc index a82ef02e32..e49663b8b0 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler.cc +++ b/tensorflow/compiler/tf2xla/xla_compiler.cc @@ -177,7 +177,9 @@ Status XlaCompiler::CompileFunction( const FunctionBody* fbody; TF_RETURN_IF_ERROR(FindFunctionBody(function, &fbody)); - TF_RETURN_IF_ERROR(CheckSignature(fbody->arg_types, args)); + TF_RETURN_WITH_CONTEXT_IF_ERROR( + CheckSignature(fbody->arg_types, args), + "Signature check failure while compiling: ", function.name()); std::unique_ptr graph(new Graph(options_.flib_def)); CopyGraph(*fbody->graph, graph.get()); -- GitLab From 7c1ff88d0e83b9b8378fb981c3a48a1ae698e12a Mon Sep 17 00:00:00 2001 From: Tayo Oguntebi <10927929+tayo@users.noreply.github.com> Date: Wed, 25 Oct 2017 09:46:40 -0700 Subject: [PATCH 351/573] Update node_def.proto Clarifying comments for valid device string in NodeDef, as discussed in PR #13874. Notes: 1. The device string is as emitted by: tensorflow/python/framework/device.py to_string() function. 2. I notice our regex convention does not use '+' (e.g. XX* instead of X+). --- tensorflow/core/framework/node_def.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto index 1fd2e50b51..8fcee32e29 100644 --- a/tensorflow/core/framework/node_def.proto +++ b/tensorflow/core/framework/node_def.proto @@ -35,7 +35,7 @@ message NodeDef { // CONSTRAINT ::= ("job:" JOB_NAME) // | ("replica:" [1-9][0-9]*) // | ("task:" [1-9][0-9]*) - // | ("device:" ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") ) + // | ("device:" [A-Za-z]* ":" ([1-9][0-9]* | "*") ) // // Valid values for this string include: // * "/job:worker/replica:0/task:1/device:GPU:3" (full specification) -- GitLab From f1f60ac3e59b7cbbd2badef11cc3da42064fc695 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 09:47:05 -0700 Subject: [PATCH 352/573] 1. Separate the special case BinaryFunctor when NDIMS == 2 into a template specialization. This prevents the NDIMS==2 optimization code from being compiled in the general case, which can lead to compile time errors when the underlying Eigen implementation becomes more strict about NDIMS. 2. Fix the 64-bit dimension() call on output in extract_image_patches_op.h when other operands have been cast to use 32-bit index. PiperOrigin-RevId: 173409602 --- tensorflow/core/kernels/cwise_ops_common.h | 75 +++++++++++++++---- .../core/kernels/extract_image_patches_op.h | 5 +- 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h index 2454620776..8295fa939e 100644 --- a/tensorflow/core/kernels/cwise_ops_common.h +++ b/tensorflow/core/kernels/cwise_ops_common.h @@ -305,6 +305,62 @@ struct BinaryFunctor { Assign(d, out, in.unaryExpr(Unary(scalar.data()))); } + void BCast(const CPUDevice& dev, + typename TTypes::Tensor out, + typename TTypes::ConstTensor in0, + typename Eigen::array bcast0, + typename TTypes::ConstTensor in1, + typename Eigen::array bcast1, + bool* error) { + typename Functor::func func; + if (AllOne(bcast0) && AllOne(bcast1)) { + Assign(dev, out, in0.binaryExpr(in1, func)); + } else if (AllOne(bcast0)) { + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, in0.binaryExpr(rhs, func)); + } else if (AllOne(bcast1)) { + auto lhs = in0.broadcast(bcast0); + Assign(dev, out, lhs.binaryExpr(in1, func)); + } else { + auto lhs = in0.broadcast(bcast0); + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, lhs.binaryExpr(rhs, func)); + } + } +}; + +// Partial specialization of BinaryFunctor +// for functors with with no error checking. +template +struct BinaryFunctor { + enum { NDIMS = 2 }; + + void operator()(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tin_type in0, + typename Functor::tin_type in1, bool* error) { + Assign(d, out, in0.binaryExpr(in1, typename Functor::func())); + } + + void Left(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tscalar_type scalar, + typename Functor::tin_type in, bool* error) { + typedef typename Functor::out_type Tout; + typedef typename Functor::in_type Tin; + typedef typename Functor::func Binary; + typedef typename Eigen::internal::scalar_left Unary; + Assign(d, out, in.unaryExpr(Unary(scalar.data()))); + } + + void Right(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tin_type in, + typename Functor::tscalar_type scalar, bool* error) { + typedef typename Functor::out_type Tout; + typedef typename Functor::in_type Tin; + typedef typename Functor::func Binary; + typedef typename Eigen::internal::scalar_right Unary; + Assign(d, out, in.unaryExpr(Unary(scalar.data()))); + } + #if !defined(EIGEN_HAS_INDEX_LIST) inline Eigen::DSizes NByOne(int n) { return Eigen::DSizes(n, 1); @@ -334,8 +390,7 @@ struct BinaryFunctor { bool* error) { typedef typename Functor::in_type T; typename Functor::func func; - if ((NDIMS == 2) && Functor::use_bcast_optimization && - use_bcast_optimization::value) { + if (Functor::use_bcast_optimization && use_bcast_optimization::value) { // Optimize for speed by using Eigen::type2index and avoid // .broadcast() when we know its a no-op. // @@ -411,19 +466,9 @@ struct BinaryFunctor { } // Fallback path. Always works and probably slower. - if (AllOne(bcast0) && AllOne(bcast1)) { - Assign(dev, out, in0.binaryExpr(in1, func)); - } else if (AllOne(bcast0)) { - auto rhs = in1.broadcast(bcast1); - Assign(dev, out, in0.binaryExpr(rhs, func)); - } else if (AllOne(bcast1)) { - auto lhs = in0.broadcast(bcast0); - Assign(dev, out, lhs.binaryExpr(in1, func)); - } else { - auto lhs = in0.broadcast(bcast0); - auto rhs = in1.broadcast(bcast1); - Assign(dev, out, lhs.binaryExpr(rhs, func)); - } + auto lhs = in0.broadcast(bcast0); + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, lhs.binaryExpr(rhs, func)); } }; diff --git a/tensorflow/core/kernels/extract_image_patches_op.h b/tensorflow/core/kernels/extract_image_patches_op.h index 9d34daca64..e430a23d20 100644 --- a/tensorflow/core/kernels/extract_image_patches_op.h +++ b/tensorflow/core/kernels/extract_image_patches_op.h @@ -34,11 +34,12 @@ struct ExtractImagePatchesForward { // NHWC format while Eigen assumes NWHC format. const int64 N = std::max(input.size(), output.size()); if (N <= std::numeric_limits::max()) { - To32Bit(output).device(d) = + auto output_32bit = To32Bit(output); + output_32bit.device(d) = To32Bit(input) .extract_image_patches(patch_cols, patch_rows, stride_cols, stride_rows, rate_cols, rate_rows, padding) - .reshape(output.dimensions()); + .reshape(output_32bit.dimensions()); } else { output.device(d) = input -- GitLab From 805594a4643dda027ff45be7edd2ab94f57a9dec Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 10:22:46 -0700 Subject: [PATCH 353/573] [XLA] Set shape of -C equal to shape of C in A/pow(B,C) -> A*pow(B,-C). PiperOrigin-RevId: 173414738 --- .../xla/service/algebraic_simplifier.cc | 9 +++-- .../xla/service/algebraic_simplifier_test.cc | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 8b3886cc7a..ae26cc2d99 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -523,11 +523,16 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide, // A/pow(B,C) => A*pow(B,-C) if (rhs->opcode() == HloOpcode::kPower) { VLOG(10) << "transform [A/pow(B,C) => A*pow(B,-C)]: " << divide->ToString(); + // The output shape of the created negate operator should be the same as the + // input. + const Shape& negate_shape = rhs->operand(1)->shape(); HloInstruction* negate = computation_->AddInstruction(HloInstruction::CreateUnary( - divide->shape(), HloOpcode::kNegate, rhs->mutable_operand(1))); + negate_shape, HloOpcode::kNegate, rhs->mutable_operand(1))); + // And the power operator should retain the output shape of the old one. + const Shape& new_power_shape = rhs->shape(); HloInstruction* new_power = computation_->AddInstruction( - HloInstruction::CreateBinary(divide->shape(), HloOpcode::kPower, + HloInstruction::CreateBinary(new_power_shape, HloOpcode::kPower, rhs->mutable_operand(0), negate)); return ReplaceWithNewInstruction( divide, HloInstruction::CreateBinary( diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index af502206e2..57be144b36 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -353,6 +353,42 @@ TEST_F(AlgebraicSimplifierTest, DivOfPower) { op::Multiply(param0, op::Power(param1, op::Negate(param2)))); } +// Test that broadcasting is done on the right step when simplifying A/pow(B,C) +// to A*pow(B,-C). +TEST_F(AlgebraicSimplifierTest, DivOfBroadcastingPower) { + Shape r0f32 = ShapeUtil::MakeShape(F32, {}); + Shape r1f32 = ShapeUtil::MakeShape(F32, {7}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r1f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r1f32, "param1")); + HloInstruction* param2 = builder.AddInstruction( + HloInstruction::CreateParameter(2, r0f32, "param2")); + HloInstruction* power = builder.AddInstruction( + HloInstruction::CreateBinary(r1f32, HloOpcode::kPower, param1, param2)); + builder.AddInstruction( + HloInstruction::CreateBinary(r1f32, HloOpcode::kDivide, param0, power)); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + + EXPECT_THAT(computation->root_instruction(), + op::Divide(param0, op::Power(param1, param2))); + + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + + ASSERT_THAT(computation->root_instruction(), + op::Multiply(param0, op::Power(param1, op::Negate(param2)))); + + const HloInstruction* negate = + computation->root_instruction()->operand(1)->operand(1); + const Shape& negate_shape = negate->shape(); + EXPECT_EQ(0, negate_shape.dimensions_size()); +} + // Test that A/1 is simplified to A for a scalar. TEST_F(AlgebraicSimplifierTest, DivOneScalar) { Shape r0f32 = ShapeUtil::MakeShape(F32, {}); -- GitLab From f75a6b866524ce877321f0ef9f3508b9b6c705fc Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 25 Oct 2017 10:29:23 -0700 Subject: [PATCH 354/573] MomentumOptimizer has two required args which causes error in optimize_loss. optimize_loss function tries to initialize Optimizer like... opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr) This does not work for MomentumOptimizer which also has required arg of 'momentum'. PiperOrigin-RevId: 173415707 --- tensorflow/contrib/layers/python/layers/optimizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/layers/python/layers/optimizers.py b/tensorflow/contrib/layers/python/layers/optimizers.py index 33db93b970..cdceea6fee 100644 --- a/tensorflow/contrib/layers/python/layers/optimizers.py +++ b/tensorflow/contrib/layers/python/layers/optimizers.py @@ -41,7 +41,7 @@ OPTIMIZER_CLS_NAMES = { "Adagrad": train.AdagradOptimizer, "Adam": train.AdamOptimizer, "Ftrl": train.FtrlOptimizer, - "Momentum": train.MomentumOptimizer, + "Momentum": lambda lr: train.MomentumOptimizer(lr, momentum=0.9), "RMSProp": train.RMSPropOptimizer, "SGD": train.GradientDescentOptimizer, } -- GitLab From 51d95238a6ccbf540bda0299548c9bbb8b304130 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 11:29:17 -0700 Subject: [PATCH 355/573] Add tests for utils.SubGraph in K-FAC PiperOrigin-RevId: 173425276 --- .../kfac/python/kernel_tests/utils_test.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py b/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py index 779a8179bb..55fe38e3e9 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py @@ -63,6 +63,39 @@ class SequenceDictTest(test.TestCase): self.assertItemsEqual(list(zip(keys, values)), seq_dict.items()) +class SubGraphTest(test.TestCase): + + def testBasicGraph(self): + a = array_ops.constant([[1., 2.], [3., 4.]]) + b = array_ops.constant([[5., 6.], [7., 8.]]) + c = a + b + d = a * b + sub_graph = utils.SubGraph((c,)) + self.assertTrue(sub_graph.is_member(a)) + self.assertTrue(sub_graph.is_member(b)) + self.assertTrue(sub_graph.is_member(c)) + self.assertFalse(sub_graph.is_member(d)) + + def testRepeatedAdds(self): + a = array_ops.constant([[1., 2.], [3., 4.]]) + b = array_ops.constant([[5., 6.], [7., 8.]]) + c = a + b + a # note that a appears twice in this graph + sub_graph = utils.SubGraph((c,)) + self.assertTrue(sub_graph.is_member(a)) + self.assertTrue(sub_graph.is_member(b)) + self.assertTrue(sub_graph.is_member(c)) + + def testFilterList(self): + a = array_ops.constant([[1., 2.], [3., 4.]]) + b = array_ops.constant([[5., 6.], [7., 8.]]) + c = a + b + d = a * b + sub_graph = utils.SubGraph((c,)) + input_list = [b, d] + filtered_list = sub_graph.filter_list(input_list) + self.assertEqual(filtered_list, [b]) + + class UtilsTest(test.TestCase): def _fully_connected_layer_params(self): -- GitLab From 7ce9b664a7422e07cdda843f7256fdb3fd454082 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Wed, 25 Oct 2017 11:37:41 -0700 Subject: [PATCH 356/573] Reduce lock contention to prevent interop threadpool deadlocks. PiperOrigin-RevId: 173426652 --- tensorflow/contrib/data/python/kernel_tests/BUILD | 3 --- tensorflow/core/kernels/map_and_batch_dataset_op.cc | 10 ++++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 5339ebb689..b8cdb7b20d 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -11,9 +11,6 @@ py_test( size = "small", srcs = ["batch_dataset_op_test.py"], srcs_version = "PY2AND3", - tags = [ - "manual", # b/67958604 - ], deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc index 332a96ae03..f9f68a5418 100644 --- a/tensorflow/core/kernels/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc @@ -287,10 +287,12 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { EXCLUSIVE_LOCKS_REQUIRED(mu_) { port::Tracing::TraceMe activity(strings::StrCat(prefix(), "::Start")); // Initialize batch result. - mutex_lock l(batch_results_[batch_index].mu); - batch_results_[batch_index].output_allocated = false; - batch_results_[batch_index].counter.reset( - new BlockingCounter(dataset()->batch_size_)); + { + mutex_lock l(batch_results_[batch_index].mu); + batch_results_[batch_index].output_allocated = false; + batch_results_[batch_index].counter.reset( + new BlockingCounter(dataset()->batch_size_)); + } // Initialize invocation results. for (size_t i = 0; i < dataset()->batch_size_; ++i) { size_t index = ComputeInvocationIndex(batch_index, i); -- GitLab From 9469522bf13933bf9f53480e63d0ef36f7013b94 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 11:54:39 -0700 Subject: [PATCH 357/573] Adds eager compatability message for Variable. PiperOrigin-RevId: 173429067 --- tensorflow/contrib/eager/python/tfe_test.py | 6 ++++++ tensorflow/python/ops/variables.py | 14 +++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index 6b5053125b..eabff7f0a8 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -39,6 +40,11 @@ class TFETest(test_util.TensorFlowTestCase): r'indices = 7 is not in \[0, 3\)'): array_ops.gather([0, 1, 2], 7) + def testVariableError(self): + with self.assertRaisesRegexp( + RuntimeError, r'Variable not supported in Eager mode'): + variables.Variable(initial_value=1.0) + def testGradients(self): def square(x): diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 0272f77176..fd0aee3c33 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -118,6 +118,14 @@ class Variable(object): `trainable_variables()` returns the contents of this collection. The various `Optimizer` classes use this collection as the default list of variables to optimize. + + @compatiblity(eager) + `tf.Variable` is not compatible with eager execution. Use + `tfe.Variable` instead which is compatable with both eager execution + and graph construction. See [the TensorFlow Eager Execution + guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) + for details on how variables work in eager execution. + @end_compatiblity """ def __init__(self, @@ -188,11 +196,11 @@ class Variable(object): ValueError: If both `variable_def` and initial_value are specified. ValueError: If the initial value is not specified, or does not have a shape and `validate_shape` is `True`. - RuntimeError: If created in EAGER mode. + RuntimeError: If eager execution is enabled. """ if not context.in_graph_mode(): - raise RuntimeError("Variable not supported in Eager mode. " - "Please use ResourceVariable instead") + raise RuntimeError("tf.Variable not supported in Eager mode. " + "Please use tfe.Variable instead") if variable_def: # If variable_def is provided, recreates the variable from its fields. if initial_value: -- GitLab From 06674161dc5c6b860bea559c2eef0217d49d86ca Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 12:11:43 -0700 Subject: [PATCH 358/573] Variable name remapping of saver/restore. PiperOrigin-RevId: 173431453 --- tensorflow/contrib/eager/python/saver.py | 42 +++++++++---- tensorflow/contrib/eager/python/saver_test.py | 40 ++++++++++++- .../core/kernels/resource_variable_ops.cc | 2 +- tensorflow/python/training/saver_test.py | 60 +++++++++++-------- 4 files changed, 102 insertions(+), 42 deletions(-) diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py index 2bf11d3f20..404f77105a 100644 --- a/tensorflow/contrib/eager/python/saver.py +++ b/tensorflow/contrib/eager/python/saver.py @@ -29,22 +29,23 @@ from tensorflow.python.training import saver as _saver def _init_from_checkpoint(self, *args, **kwargs): """Overrides default init by loading value from checkpoint.""" - self.old_init(*args, **kwargs) # pylint: disable=protected-access - if self._shared_name not in self.ckpt_var_cache: + self._old_init(*args, **kwargs) + ckpt_name = self._map_func(self._shared_name) + if ckpt_name not in self._ckpt_var_cache: raise errors.NotFoundError(None, None, - "%s not found in checkpoint" % self._shared_name) + "%s not found in checkpoint" % ckpt_name) - val = self.ckpt_var_cache[self._shared_name] + val = self._ckpt_var_cache.get(ckpt_name, None) if val is not None: - self.assign(self.ckpt_var_cache[self._shared_name]) + self.assign(val) # Avoid assigning for the second time. - self.ckpt_var_cache[self._shared_name] = None + self._ckpt_var_cache[ckpt_name] = None # pylint: enable=protected-access @contextlib.contextmanager -def restore_variables_on_create(save_path): +def restore_variables_on_create(save_path, map_func=None): """ContextManager that restores variables on creation. When save_path is None (e.g. No checkpoint), does nothing. @@ -59,19 +60,31 @@ def restore_variables_on_create(save_path): Args: save_path: The checkpoint file prefix. + map_func: A function that given the variable name as argument + and returns a variable name in checkpoint for restore. If + None, use the variable with the same name in checkpoint to restore. + It's an error that the mapped variable name doesn't exist in + checkpoint. Yields: Nothing. Raises: NotFoundError: If the variable is not found in checkpoint. - ValueError: If not used in eager mode. + ValueError: If not used in eager mode or map_func is not callable. """ if context.in_graph_mode(): raise ValueError( "Currently, restore_variables_on_create can only be used with " "eager execution enabled.") if save_path: + if map_func is None: + map_func_wrapper = lambda self, x: x + else: + if not callable(map_func): + raise ValueError("map_func must be callaled.") + map_func_wrapper = lambda self, x: map_func(x) + ckpt_var_cache = dict() reader = checkpoint_utils.load_checkpoint(save_path) for k, _ in checkpoint_utils.list_variables(save_path): @@ -82,8 +95,10 @@ def restore_variables_on_create(save_path): assert old_init, "ResourceVariable misses _init_from_args method." setattr(resource_variable_ops.ResourceVariable, "_init_from_args", _init_from_checkpoint) - setattr(resource_variable_ops.ResourceVariable, "old_init", old_init) - setattr(resource_variable_ops.ResourceVariable, "ckpt_var_cache", + setattr(resource_variable_ops.ResourceVariable, "_old_init", old_init) + setattr(resource_variable_ops.ResourceVariable, "_map_func", + map_func_wrapper) + setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", ckpt_var_cache) try: yield @@ -93,8 +108,9 @@ def restore_variables_on_create(save_path): if save_path: setattr(resource_variable_ops.ResourceVariable, "_init_from_args", old_init) - setattr(resource_variable_ops.ResourceVariable, "old_init", None) - setattr(resource_variable_ops.ResourceVariable, "ckpt_var_cache", None) + setattr(resource_variable_ops.ResourceVariable, "_old_init", None) + setattr(resource_variable_ops.ResourceVariable, "_map_func", None) + setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", None) class Saver(object): @@ -104,7 +120,7 @@ class Saver(object): session is not needed. Args: - var_list: A list of variables. + var_list: Same as tf.train.Saver. """ def __init__(self, var_list): diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py index 1605435d8d..3c69b90242 100644 --- a/tensorflow/contrib/eager/python/saver_test.py +++ b/tensorflow/contrib/eager/python/saver_test.py @@ -55,7 +55,7 @@ class SaverTest(test.TestCase): self.assertEqual(v1.read_value().numpy(), 1.0) def testSameNameNoClobbering(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): # Note that this test purposefully uses Graphs rather than # IsolateTest. Users are more likely to accidentally create the same # variable name this way. @@ -70,7 +70,7 @@ class SaverTest(test.TestCase): saver.save(ckpt_prefix) def testDifferentGraphError(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): with ops.Graph().as_default(): v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') with ops.Graph().as_default(): @@ -80,7 +80,7 @@ class SaverTest(test.TestCase): saver.save(ckpt_prefix) def testSameObjectOK(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') # While different objects with the same shared_name are not good, passing # in the same object multiple times is fine. @@ -88,6 +88,40 @@ class SaverTest(test.TestCase): ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') saver.save(ckpt_prefix) + def testSaveByDict(self): + with ops.device(self._dev()): + v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') + v2 = resource_variable_ops.ResourceVariable(1.0, name='v2') + def model(): + return array_ops.constant(2.0) * v1 * v2 + + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + + # Save the variables under different names. + _ = model() + saver = _saver.Saver({'ckpt/v1': v1, 'ckpt/v2': v2}) + saver.save(ckpt_prefix) + v1.assign(2.0) + v2.assign(2.0) + self.assertEqual(v1.read_value().numpy(), 2.0) + self.assertEqual(v2.read_value().numpy(), 2.0) + # Can still restore it. + saver.restore(ckpt_prefix) + self.assertEqual(v1.read_value().numpy(), 1.0) + self.assertEqual(v1.read_value().numpy(), 1.0) + # However, cannot restore it with default name. + with self.assertRaisesOpError('not found in checkpoint'): + saver = _saver.Saver([v1, v2]).restore(ckpt_prefix) + + # Can specify which variable in ckpt to restore to which variable. + def map_func(x): + return {'v3': 'ckpt/v1', 'v4': 'ckpt/v2'}.get(x, x) + with _saver.restore_variables_on_create(ckpt_prefix, map_func): + v3 = resource_variable_ops.ResourceVariable(2.0, name='v3') + v4 = resource_variable_ops.ResourceVariable(2.0, name='v4') + self.assertEqual(v3.read_value().numpy(), 1.0) + self.assertEqual(v4.read_value().numpy(), 1.0) + def testRestoreOnCreate(self): with ops.device(self._dev()): def model(init_val): diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index 90db0c2b7b..a4db4abd7b 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -85,7 +85,7 @@ class ReadVariableOp : public OpKernel { errors::NotFound( "Error while reading resource variable ", handle.name(), " from Container: ", handle.container(), - ". This could mean that the variable was not initialized. ", + ". This could mean that the variable was uninitialized. ", status.ToString())); core::ScopedUnref s(variable); diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index 4abff1d106..744b17dd22 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -1299,20 +1299,20 @@ class KeepCheckpointEveryNHoursTest(test.TestCase): class SaveRestoreWithVariableNameMap(test.TestCase): - def testNonReshape(self): + def _testNonReshape(self, variable_op): save_path = os.path.join(self.get_temp_dir(), "non_reshape") - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: # Build a graph with 2 parameter nodes, and Save and # Restore nodes for them. - v0 = variables.Variable(10.0, name="v0") - v1 = variables.Variable(20.0, name="v1") + v0 = variable_op(10.0, name="v0") + v1 = variable_op(20.0, name="v1") save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1}) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) # Check that the parameter nodes have been initialized. - self.assertEqual(10.0, v0.eval()) - self.assertEqual(20.0, v1.eval()) + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) # Save the initialized values in the file at "save_path" # Use a variable name map to set the saved tensor names @@ -1327,40 +1327,50 @@ class SaveRestoreWithVariableNameMap(test.TestCase): # Verify that the mapped names are present in the Saved file and can be # Restored using remapped names. - with self.test_session() as sess: - v0 = variables.Variable(-1.0, name="v0") - v1 = variables.Variable(-1.0, name="v1") + with self.test_session(graph=ops_lib.Graph()) as sess: + v0 = variable_op(-1.0, name="v0") + v1 = variable_op(-1.0, name="v1") - with self.assertRaisesOpError("uninitialized value v0"): - sess.run(v0) - with self.assertRaisesOpError("uninitialized value v1"): - sess.run(v1) + if context.in_graph_mode(): + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v0) + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v1) save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1}) save.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, v0.eval()) - self.assertEqual(20.0, v1.eval()) + if context.in_graph_mode(): + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) # Add a prefix to the node names in the current graph and Restore using # remapped names. - with self.test_session() as sess: - v0 = variables.Variable(-1.0, name="restore_prefix/v0") - v1 = variables.Variable(-1.0, name="restore_prefix/v1") + with self.test_session(graph=ops_lib.Graph()) as sess: + v0 = variable_op(-1.0, name="restore_prefix/v0") + v1 = variable_op(-1.0, name="restore_prefix/v1") - with self.assertRaisesOpError("uninitialized value restore_prefix/v0"): - sess.run(v0) - with self.assertRaisesOpError("uninitialized value restore_prefix/v1"): - sess.run(v1) + if context.in_graph_mode(): + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v0) + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v1) # Restore the saved values in the parameter nodes. save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1}) save.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, v0.eval()) - self.assertEqual(20.0, v1.eval()) + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) + + @test_util.run_in_graph_and_eager_modes() + def testNonReshapeResourceVariable(self): + self._testNonReshape(resource_variable_ops.ResourceVariable) + + def testNonReshapeVariable(self): + self._testNonReshape(variables.Variable) class LatestCheckpointWithRelativePaths(test.TestCase): -- GitLab From df3b4444ccb61ef4858a6acf195d93f9c43d7b24 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Wed, 25 Oct 2017 12:22:02 -0700 Subject: [PATCH 359/573] Dedent compatibility blocks This is to avoid accidentally triggering markdown's 4-space code-formatting PiperOrigin-RevId: 173432680 --- tensorflow/tools/docs/pretty_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/docs/pretty_docs.py b/tensorflow/tools/docs/pretty_docs.py index 5ea9394865..92f50189dd 100644 --- a/tensorflow/tools/docs/pretty_docs.py +++ b/tensorflow/tools/docs/pretty_docs.py @@ -290,7 +290,9 @@ def _build_compatibility(compatibility): for key in sorted_keys: value = compatibility[key] - parts.append('\n\n#### %s compatibility\n%s\n' % (key, value)) + # Dedent so that it does not trigger markdown code formatting. + value = textwrap.dedent(value) + parts.append('\n\n#### %s Compatibility\n%s\n' % (key.title(), value)) return ''.join(parts) -- GitLab From 9ad48ae2652291b4ea6f4b03da8637f58d6fccf4 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Wed, 25 Oct 2017 13:37:14 -0700 Subject: [PATCH 360/573] Add de-fuser pass which replaces fusion nodes with their non-fused equivalents. This pass will be used in tools added in followup CLs. The motivation for this pass is running an HLO module which was built for one backend on a different backend for testing and debugging. De-fusing is necessary because different backends have different fusion support. Also, fix problem with deletion of fused computation identified when testing the defuser. PiperOrigin-RevId: 173442671 --- tensorflow/compiler/xla/service/BUILD | 27 +++ tensorflow/compiler/xla/service/defuser.cc | 115 ++++++++++ tensorflow/compiler/xla/service/defuser.h | 41 ++++ .../compiler/xla/service/defuser_test.cc | 214 ++++++++++++++++++ 4 files changed, 397 insertions(+) create mode 100644 tensorflow/compiler/xla/service/defuser.cc create mode 100644 tensorflow/compiler/xla/service/defuser.h create mode 100644 tensorflow/compiler/xla/service/defuser_test.cc diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 8f5105aa53..fe5889efe1 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1074,6 +1074,33 @@ tf_cc_test( ], ) +cc_library( + name = "defuser", + srcs = ["defuser.cc"], + hdrs = ["defuser.h"], + deps = [ + ":call_graph", + ":hlo", + ":hlo_pass", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "defuser_test", + srcs = ["defuser_test.cc"], + deps = [ + ":defuser", + ":hlo_matchers", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla/tests:hlo_verified_test_base", + ], +) + cc_library( name = "tuple_simplifier", srcs = ["tuple_simplifier.cc"], diff --git a/tensorflow/compiler/xla/service/defuser.cc b/tensorflow/compiler/xla/service/defuser.cc new file mode 100644 index 0000000000..d124f74d19 --- /dev/null +++ b/tensorflow/compiler/xla/service/defuser.cc @@ -0,0 +1,115 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/defuser.h" + +#include +#include +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/service/call_graph.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +namespace { + +// Copy all the instructions in the given fusion instruction into the fusion +// instruction's parent computation and replace the use of the fusion +// instruction with the copy of the fusion expression root. +Status Defuse(HloInstruction* fusion_instruction) { + VLOG(2) << "Defusing instruction: " << fusion_instruction->ToString(); + + HloComputation* fused_computation = + fusion_instruction->fused_instructions_computation(); + + // A map from fused instruction to its defused clone. + tensorflow::gtl::FlatMap + defused_instructions; + // Initialize map to contain the fusion instruction parameters mapping + // to the operands of the fusion instruction. + for (int64 i = 0; i < fusion_instruction->operand_count(); ++i) { + defused_instructions[fused_computation->parameter_instruction(i)] = + fusion_instruction->mutable_operand(i); + } + + // Create a clone of each instruction of the fused computation in the same + // computation as the fusion instruction itself. + // TODO(b/68227302): Moving instruction to new computation rather than + // cloning and deleting. + for (HloInstruction* fused_instruction : + fused_computation->MakeInstructionPostOrder()) { + if (fused_instruction->opcode() == HloOpcode::kParameter) { + continue; + } + std::vector new_operands; + for (HloInstruction* operand : fused_instruction->operands()) { + new_operands.push_back(defused_instructions.at(operand)); + } + HloInstruction* defused_instruction = + fusion_instruction->parent()->AddInstruction( + fused_instruction->CloneWithNewOperands(fused_instruction->shape(), + new_operands)); + defused_instructions[fused_instruction] = defused_instruction; + } + + TF_RETURN_IF_ERROR(fusion_instruction->ReplaceAllUsesWith( + defused_instructions.at(fusion_instruction->fused_expression_root()))); + + HloModule* module = fusion_instruction->parent()->parent(); + TF_RETURN_IF_ERROR( + fusion_instruction->parent()->RemoveInstruction(fusion_instruction)); + return module->RemoveEmbeddedComputation(fused_computation); +} + +} // namespace + +StatusOr Defuser::Run(HloModule* module) { + VLOG(1) << "Defusing module " << module->name(); + XLA_VLOG_LINES(2, "Before defusion:\n" + module->ToString()); + + bool changed = false; + std::unique_ptr call_graph = CallGraph::Build(module); + TF_RETURN_IF_ERROR(call_graph->VisitNodes( + [&](const CallGraphNode& call_graph_node) -> Status { + if (call_graph_node.computation()->IsFusionComputation()) { + TF_RET_CHECK(call_graph_node.caller_callsites().size() == 1); + HloInstruction* fusion_instruction = + call_graph_node.caller_callsites()[0].instruction(); + TF_RETURN_IF_ERROR(Defuse(fusion_instruction)); + changed = true; + } + return Status::OK(); + }, + /*visit_unreachable_nodes=*/true)); + + XLA_VLOG_LINES(2, "After defusion:\n" + module->ToString()); + + return changed; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/defuser.h b/tensorflow/compiler/xla/service/defuser.h new file mode 100644 index 0000000000..56b28fd22d --- /dev/null +++ b/tensorflow/compiler/xla/service/defuser.h @@ -0,0 +1,41 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_ + +#include + +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" + +namespace xla { + +// A pass which replaces all fusion instructions with the equivalent un-fused +// instructions. +class Defuser : public HloPassInterface { + public: + Defuser() {} + ~Defuser() override {} + tensorflow::StringPiece name() const override { return "defuser"; } + + // Run defusion on the given module. Returns whether the module was + // changed. + StatusOr Run(HloModule* module) override; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_ diff --git a/tensorflow/compiler/xla/service/defuser_test.cc b/tensorflow/compiler/xla/service/defuser_test.cc new file mode 100644 index 0000000000..32b5c5d35f --- /dev/null +++ b/tensorflow/compiler/xla/service/defuser_test.cc @@ -0,0 +1,214 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/defuser.h" + +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h" + +namespace op = xla::testing::opcode_matchers; + +namespace xla { +namespace { + +class DefuserTest : public HloVerifiedTestBase { + protected: + // Returns the number of fusion instructions in the module. + int FusionCount() { + int count = 0; + for (HloComputation* computation : module().computations()) { + if (computation->IsFusionComputation()) { + count++; + } + } + return count; + } + + Defuser defuser_; + const Shape shape_ = ShapeUtil::MakeShape(F32, {2, 2}); +}; + +TEST_F(DefuserTest, NoFusionInstruction) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + + module().AddEntryComputation(builder.Build()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_FALSE(defuser_.Run(&module()).ValueOrDie()); +} + +TEST_F(DefuserTest, TrivialFusionInstructionAsRoot) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction({add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(1, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Add(op::Parameter(), op::Parameter())); +} + +TEST_F(DefuserTest, TrivialFusionInstructionNotAsRoot) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction({add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Negate(op::Fusion())); + + EXPECT_EQ(1, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Negate(op::Add(op::Parameter(), op::Parameter()))); +} + +TEST_F(DefuserTest, NonTrivialFusionInstruction) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto param3 = + builder.AddInstruction(HloInstruction::CreateParameter(2, shape_, "p2")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + auto negate = builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + auto sub = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kSubtract, add, negate)); + auto mul = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kMultiply, sub, param3)); + auto div = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kDivide, mul, param3)); + auto constant = builder.AddInstruction(HloInstruction::CreateConstant( + Literal::CreateR2({{1.0, 2.0}, {3.0, 4.0}}))); + auto add2 = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, constant, div)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction( + {add2, constant, div, mul, sub, negate, add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(1, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Add(op::Constant(), op::Divide())); +} + +TEST_F(DefuserTest, MultipleFusionInstructions) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto param3 = + builder.AddInstruction(HloInstruction::CreateParameter(2, shape_, "p2")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + auto negate = builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + auto sub = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kSubtract, add, negate)); + auto mul = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kMultiply, sub, param3)); + auto div = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kDivide, mul, param3)); + auto constant = builder.AddInstruction(HloInstruction::CreateConstant( + Literal::CreateR2({{1.0, 2.0}, {3.0, 4.0}}))); + auto add2 = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, constant, div)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction({add2, constant, div, mul}, + HloInstruction::FusionKind::kLoop); + computation->CreateFusionInstruction({sub, negate, add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(2, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Add(op::Constant(), op::Divide())); +} + +TEST_F(DefuserTest, NestedFusionInstructions) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + auto negate = builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + + auto computation = module().AddEntryComputation(builder.Build()); + auto outer_fusion = computation->CreateFusionInstruction( + {negate, add}, HloInstruction::FusionKind::kLoop); + HloInstruction* fused_negate = outer_fusion->fused_expression_root(); + ASSERT_EQ(fused_negate->opcode(), HloOpcode::kNegate); + outer_fusion->fused_instructions_computation()->CreateFusionInstruction( + {fused_negate}, HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(2, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), op::Negate(op::Add())); +} + +} // namespace +} // namespace xla -- GitLab From 772bebb16a6c359021ff789f89c6b7dfdd0d3126 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 13:53:46 -0700 Subject: [PATCH 361/573] Better error message for ResourceVariable.set_shape PiperOrigin-RevId: 173445012 --- tensorflow/python/ops/resource_variable_ops.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 06c5a3bb2a..386fd204b6 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -680,6 +680,10 @@ class ResourceVariable(variables.Variable): """Unsupported.""" raise NotImplementedError("ResourceVariable does not implement _ref()") + def set_shape(self, shape): + """Unsupported.""" + raise NotImplementedError("ResourceVariable does not implement set_shape()") + @staticmethod def _OverloadOperator(operator): # pylint: disable=invalid-name """Defer an operator overload to `ops.Tensor`. -- GitLab From 433928745b5a76972a6128fe9b89f19b7a5e2d6c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:02:16 -0700 Subject: [PATCH 362/573] Updated hlo parser tests to test "device=". PiperOrigin-RevId: 173446328 --- .../compiler/xla/tools/parser/hlo_parser_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc index 5150e1f96d..2bf1cce1c0 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -100,8 +100,8 @@ ENTRY %add_constants () -> f32[] { R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module: ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { - %v1 = f32[4]{0} parameter(0) - %v2 = f32[4]{0} parameter(1) + %v1 = f32[4]{0} parameter(0), device=1 + %v2 = f32[4]{0} parameter(1), device=1 %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2) ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) } @@ -164,9 +164,9 @@ ENTRY %WhileWithScalarS32Result.v2 () -> s32[] { R"(HloModule TwoSendRecvBothWayRecvFist_module: ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> f32[] { - %recv = f32[] recv(), channel_id=15 - ROOT %constant = f32[] constant(2.1) - %send = () send(f32[] %constant), channel_id=16 + %recv = f32[] recv(), channel_id=15, device=1 + ROOT %constant = f32[] constant(2.1), device=0 + %send = () send(f32[] %constant), channel_id=16, device=0 } )" @@ -180,7 +180,7 @@ ENTRY %GetTupleElement.v4 () -> s32[] { %constant = f32[] constant(1.23) %constant.1 = s32[] constant(4) %tuple = (f32[], s32[]) tuple(f32[] %constant, s32[] %constant.1) - ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1 + ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1, device=0 } )" -- GitLab From b81c8eda6074d17142b1f6e64ab30b341a8338a0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:04:56 -0700 Subject: [PATCH 363/573] Better shape function for fixed width histogram op. PiperOrigin-RevId: 173446796 --- tensorflow/core/ops/math_ops.cc | 9 ++++++++- tensorflow/python/ops/histogram_ops_test.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 61db896c51..130e3ed781 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -2258,7 +2258,14 @@ REGISTER_OP("HistogramFixedWidth") .Attr("T: {int32, int64, float32, float64}") .Attr("dtype: {int32, int64} = DT_INT32") .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->UnknownShapeOfRank(1)); + const Tensor* nbins_input = c->input_tensor(2); + if (nbins_input != nullptr) { + int64 nbins; + TF_RETURN_IF_ERROR(c->GetScalarFromTensor(nbins_input, &nbins)); + c->set_output(0, c->Vector(nbins)); + } else { + c->set_output(0, c->UnknownShapeOfRank(1)); + } return Status::OK(); }) .Doc(R"doc( diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index bf6e0296f6..19ad6cd2ba 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.ops import histogram_ops from tensorflow.python.platform import test @@ -75,6 +76,24 @@ class HistogramFixedWidthTest(test.TestCase): self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) + def test_shape_inference(self): + value_range = [0.0, 5.0] + values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]] + expected_bin_counts = [2, 1, 1, 0, 2] + placeholder = array_ops.placeholder(dtypes.int32) + with self.test_session(use_gpu=True): + hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) + self.assertAllEqual(hist.shape.as_list(), (5,)) + self.assertEqual(dtypes.int32, hist.dtype) + self.assertAllClose(expected_bin_counts, hist.eval()) + + hist = histogram_ops.histogram_fixed_width(values, value_range, + nbins=placeholder) + self.assertEquals(hist.shape.ndims, 1) + self.assertIs(hist.shape[0].value, None) + self.assertEqual(dtypes.int32, hist.dtype) + self.assertAllClose(expected_bin_counts, hist.eval({placeholder: 5})) + if __name__ == '__main__': test.main() -- GitLab From 2b95f70b25101a02befd6495370cb16aaa722f6a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:05:02 -0700 Subject: [PATCH 364/573] Show an error message in _einsum_reduction ValueError. If `tf.einsum()` is called and the length of the axis labels do not match the rank of the tensor, the ValueError now states what the problem is, which tensor is causing it, and what the rank and length of the labels are. RELNOTES: n/a PiperOrigin-RevId: 173446812 --- tensorflow/python/ops/special_math_ops.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py index 87561cff92..fe3f734322 100644 --- a/tensorflow/python/ops/special_math_ops.py +++ b/tensorflow/python/ops/special_math_ops.py @@ -251,9 +251,13 @@ def _einsum_reduction(t0, t0_axis_labels, t1, t1_axis_labels, axes_to_sum): `t1_axis_labels`. """ if len(t0_axis_labels) != len(t0.get_shape()): - raise ValueError() + raise ValueError( + 'Tensor t0 of rank %d does not match einsum reduction of length %d' % + (len(t0.get_shape()), len(t0_axis_labels))) if len(t1_axis_labels) != len(t1.get_shape()): - raise ValueError() + raise ValueError( + 'Tensor t1 of rank %d does not match einsum reduction of length %d' % + (len(t1.get_shape()), len(t1_axis_labels))) # This function computes the result of a two-argument einsum() using batch # matrix multiplication. This involves -- GitLab From 64d3df30985a523d2dcd6b76a37a1fb420d2f2aa Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:05:09 -0700 Subject: [PATCH 365/573] Adds loss_fn argument to multi_label_head. PiperOrigin-RevId: 173446829 --- tensorflow/contrib/estimator/BUILD | 2 + .../estimator/python/estimator/head.py | 84 +++++++++++++++++-- .../estimator/python/estimator/head_test.py | 80 ++++++++++++++++++ 3 files changed, 158 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 89c26d1d2f..8a7d67b5c2 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -134,6 +134,7 @@ py_library( "//tensorflow/python/estimator:metric_keys", "//tensorflow/python/estimator:model_fn", "//tensorflow/python/estimator:prediction_keys", + "//tensorflow/python/estimator:util", "//tensorflow/python/ops/losses", "//tensorflow/python/saved_model:signature_constants", ], @@ -154,6 +155,7 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", "//tensorflow/python:training", diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index d01b30d7f9..189f098005 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.estimator import model_fn +from tensorflow.python.estimator import util from tensorflow.python.estimator.canned import head as head_lib from tensorflow.python.estimator.canned import metric_keys from tensorflow.python.estimator.canned import prediction_keys @@ -147,6 +148,7 @@ def multi_label_head(n_classes, weight_column=None, thresholds=None, label_vocabulary=None, + loss_fn=None, name=None): """Creates a `_Head` for multi-label classification. @@ -158,6 +160,12 @@ def multi_label_head(n_classes, multi-hot tensor of shape `[batch_size, n_classes]`, or as an integer `SparseTensor` of class indices. + Also supports custom `loss_fn`. `loss_fn` takes `(labels, logits)` or + `(labels, logits, features)` as arguments and returns unreduced loss with + shape `[batch_size, 1]`. `loss_fn` must support indicator `labels` with shape + `[batch_size, n_classes]`. Namely, the head applies `label_vocabulary` to the + input labels before passing them to `loss_fn`. + Args: n_classes: Number of classes, must be greater than 1 (for 1 class, use `binary_classification_head`). @@ -174,6 +182,7 @@ def multi_label_head(n_classes, [0, n_classes) or multi-hot Tensor. If given, labels must be SparseTensor string type and have any value in `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. + loss_fn: Optional loss function. name: name of the head. If provided, summary and metrics keys will be suffixed by `"/" + name`. Also used as `name_scope` when creating ops. @@ -201,9 +210,11 @@ def multi_label_head(n_classes, raise ValueError( 'Length of label_vocabulary must be n_classes ({}). ' 'Given: {}'.format(n_classes, len(label_vocabulary))) + if loss_fn: + _validate_loss_fn_args(loss_fn) return _MultiLabelHead( n_classes=n_classes, weight_column=weight_column, thresholds=thresholds, - label_vocabulary=label_vocabulary, name=name) + label_vocabulary=label_vocabulary, loss_fn=loss_fn, name=name) class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access @@ -214,11 +225,13 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access weight_column=None, thresholds=None, label_vocabulary=None, + loss_fn=None, name=None): self._n_classes = n_classes self._weight_column = weight_column self._thresholds = thresholds self._label_vocabulary = label_vocabulary + self._loss_fn = loss_fn self._name = name @property @@ -263,14 +276,19 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access def create_loss(self, features, mode, logits, labels): """See `Head`.""" - del mode, features # Unused for this head. + del mode # Unused for this head. processed_labels = self._process_labels(labels) - unweighted_loss = losses.sigmoid_cross_entropy( - multi_class_labels=processed_labels, logits=logits, - reduction=losses.Reduction.NONE) - # Averages loss over classes. - unweighted_loss = math_ops.reduce_mean( - unweighted_loss, axis=-1, keep_dims=True) + if self._loss_fn: + unweighted_loss = _call_loss_fn( + loss_fn=self._loss_fn, labels=processed_labels, logits=logits, + features=features) + else: + unweighted_loss = losses.sigmoid_cross_entropy( + multi_class_labels=processed_labels, logits=logits, + reduction=losses.Reduction.NONE) + # Averages loss over classes. + unweighted_loss = math_ops.reduce_mean( + unweighted_loss, axis=-1, keep_dims=True) return head_lib.LossAndLabels( unweighted_loss=unweighted_loss, processed_labels=processed_labels) @@ -386,3 +404,53 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access threshold=threshold, name=recall_key)) return metric_ops + + +def _validate_loss_fn_args(loss_fn): + """Validates loss_fn arguments. + + Required arguments: labels, logits. + Optional arguments: features. + + Args: + loss_fn: The loss function. + Raises: + ValueError: If the signature is unexpected. + """ + loss_fn_args = util.fn_args(loss_fn) + for required_arg in ['labels', 'logits']: + if required_arg not in loss_fn_args: + raise ValueError( + 'loss_fn must contain argument: {}. ' + 'Given arguments: {}'.format(required_arg, loss_fn_args)) + invalid_args = list(set(loss_fn_args) - set(['labels', 'logits', 'features'])) + if invalid_args: + raise ValueError('loss_fn has unexpected args: {}'.format(invalid_args)) + + +def _call_loss_fn(loss_fn, labels, logits, features): + """Calls loss_fn and checks the returned shape. + + Args: + loss_fn: The loss function. + labels: Processed labels Tensor. + logits: Logits Tensor of shape [batch_size, logits_dimension]. + features: Features dict. + Returns: + Loss Tensor with shape [batch_size, 1]. + """ + loss_fn_args = util.fn_args(loss_fn) + kwargs = {} + if 'features' in loss_fn_args: + kwargs['features'] = features + unweighted_loss = loss_fn(labels=labels, logits=logits, **kwargs) + batch_size = array_ops.shape(logits)[0] + loss_shape = array_ops.shape(unweighted_loss) + check_shape_op = control_flow_ops.Assert( + math_ops.reduce_all(math_ops.equal(loss_shape, [batch_size, 1])), + data=[ + 'loss_fn must return Tensor of shape [batch_size, 1]. Given: ', + loss_shape]) + with ops.control_dependencies([check_shape_op]): + return array_ops.identity(unweighted_loss) + diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py index b7252f93ee..db7d96d508 100644 --- a/tensorflow/contrib/estimator/python/estimator/head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/head_test.py @@ -33,6 +33,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test from tensorflow.python.saved_model import signature_constants @@ -131,6 +132,37 @@ class MultiLabelHead(test.TestCase): r'Length of label_vocabulary must be n_classes \(3\). Given: 2'): head_lib.multi_label_head(n_classes=3, label_vocabulary=['foo', 'bar']) + def test_loss_fn_arg_labels_missing(self): + def _loss_fn(logits): + del logits # Unused + with self.assertRaisesRegexp( + ValueError, + r'loss_fn must contain argument: labels\. ' + r'Given arguments: \(\'logits\',\)'): + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + + def test_loss_fn_arg_logits_missing(self): + def _loss_fn(labels): + del labels # unused + with self.assertRaisesRegexp( + ValueError, + r'loss_fn must contain argument: logits\. ' + r'Given arguments: \(\'labels\',\)'): + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + + def test_loss_fn_arg_features_ok(self): + def _loss_fn(labels, logits, features): + del labels, logits, features # Unused + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + + def test_loss_fn_arg_invalid(self): + def _loss_fn(labels, logits, name=None): + del labels, logits, name # Unused + with self.assertRaisesRegexp( + ValueError, + r'loss_fn has unexpected args: \[\'name\'\]'): + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + def test_name(self): head = head_lib.multi_label_head(n_classes=4, name='foo') self.assertEqual('foo', head.name) @@ -291,6 +323,54 @@ class MultiLabelHead(test.TestCase): actual_unweighted_loss.eval( {labels_placeholder: np.array([1, 1], dtype=np.int64)}) + def test_eval_create_loss_loss_fn(self): + """Tests head.create_loss for eval mode and custom loss_fn.""" + loss = np.array([[1.], [2.]], dtype=np.float32) + logits_input = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32) + labels_input = np.array([[1, 0], [1, 1]], dtype=np.int64) + def _loss_fn(labels, logits): + check_labels = control_flow_ops.Assert( + math_ops.reduce_all(math_ops.equal(labels, labels_input)), + data=[labels]) + check_logits = control_flow_ops.Assert( + math_ops.reduce_all(math_ops.equal(logits, logits_input)), + data=[logits]) + with ops.control_dependencies([check_labels, check_logits]): + return constant_op.constant(loss) + head = head_lib.multi_label_head(n_classes=2, loss_fn=_loss_fn) + + actual_unweighted_loss, _ = head.create_loss( + features={'x': np.array(((42,),), dtype=np.int32)}, + mode=model_fn.ModeKeys.EVAL, + logits=logits_input, + labels=labels_input) + with self.test_session(): + _initialize_variables(self, monitored_session.Scaffold()) + self.assertAllClose(loss, actual_unweighted_loss.eval()) + + def test_eval_create_loss_loss_fn_wrong_shape(self): + """Tests custom loss_fn that returns Tensor of unexpected shape.""" + loss = np.array([1., 2.], dtype=np.float32) + def _loss_fn(labels, logits): + del labels, logits # Unused + return constant_op.constant(loss) + head = head_lib.multi_label_head(n_classes=2, loss_fn=_loss_fn) + + logits = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32) + labels = np.array([[1, 0], [1, 1]], dtype=np.int64) + actual_unweighted_loss, _ = head.create_loss( + features={'x': np.array(((42,),), dtype=np.int32)}, + mode=model_fn.ModeKeys.EVAL, + logits=logits, + labels=labels) + with self.test_session(): + _initialize_variables(self, monitored_session.Scaffold()) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r'loss_fn must return Tensor of shape \[batch_size, 1\]\. ' + r'Given: \] \[2\]'): + actual_unweighted_loss.eval() + def test_eval_labels_none(self): """Tests that error is raised when labels is None.""" head = head_lib.multi_label_head(n_classes=2) -- GitLab From 7492e88c2c4899da56921b09ecfaf3bd18ef2995 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:08:31 -0700 Subject: [PATCH 366/573] FusedConv2DBiasActivation never uses GEMM, so doesn't need a subclass of NodeProcessor. PiperOrigin-RevId: 173447332 --- tensorflow/core/grappler/optimizers/layout_optimizer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index 11cab8099a..b364446ad7 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -54,6 +54,7 @@ std::set GetOpsFormatSupported() { "BiasAddGrad", "FusedBatchNorm", "FusedBatchNormGrad", + "FusedConv2DBiasActivation", "MaxPool", "MaxPoolGrad"}; return ops_format_supported; -- GitLab From fd42ab3cb32ca3a6ef72dc9efb674134f264a58b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:09:48 -0700 Subject: [PATCH 367/573] Adds round_mode to QuantizeV2 op to select rounding algorithm. Options are half-away-from-zero and half-to-even. round_mode=HALF_TO_EVEN currently only applies when mode="SCALED". PiperOrigin-RevId: 173447503 --- tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/quantize_op.cc | 72 ++++++++++++++------ tensorflow/core/kernels/quantize_op_test.cc | 61 +++++++++++++++++ tensorflow/core/ops/array_ops.cc | 9 ++- tensorflow/python/ops/array_ops.py | 21 ++++++ tensorflow/tools/api/golden/tensorflow.pbtxt | 2 +- 6 files changed, 143 insertions(+), 23 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 92a0dbd0ab..277b21f833 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -4869,6 +4869,7 @@ tf_kernel_library( deps = [ ":concat_lib_hdrs", ":conv_ops", + ":cwise_op", ":eigen_helpers", ":image_resizer_state", ":ops_util", diff --git a/tensorflow/core/kernels/quantize_op.cc b/tensorflow/core/kernels/quantize_op.cc index fd34e13c29..75aa47cd6b 100644 --- a/tensorflow/core/kernels/quantize_op.cc +++ b/tensorflow/core/kernels/quantize_op.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/type_traits.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/cwise_ops.h" #include "tensorflow/core/kernels/meta_support.h" #include "tensorflow/core/kernels/quantization_utils.h" #include "tensorflow/core/lib/core/errors.h" @@ -31,6 +32,19 @@ enum { QUANTIZE_MODE_MIN_FIRST, QUANTIZE_MODE_SCALED, }; +enum { + // Round half away from zero: if the fraction of y is exactly 0.5, then + // round(y) = y + 0.5 if y > 0 + // round(y) = y - 0.5 if y < 0 + // E.g., -5.5 gets rounded to -6, -5.4 goes to -5, + // 5.4 goes to 5, and 5.5 goes to 6. + ROUND_HALF_AWAY_FROM_ZERO, + // Round half to even: if the fraction of y is exactly 0.5, then round(y) is + // the nearest even integer to y. + // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes + // -24, and -24.5 gets rounded to 24. + ROUND_HALF_TO_EVEN, +}; } // namespace namespace tensorflow { @@ -66,6 +80,26 @@ class QuantizeV2Op : public OpKernel { } else if (mode_string == "SCALED") { mode_ = QUANTIZE_MODE_SCALED; } + + string round_mode_string; + OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string)); + OP_REQUIRES(ctx, + (round_mode_string == "HALF_AWAY_FROM_ZERO" || + round_mode_string == "HALF_TO_EVEN"), + errors::InvalidArgument("Round mode string must be " + "'HALF_AWAY_FROM_ZERO' or " + "'HALF_TO_EVEN', is '" + + round_mode_string + "'")); + if (round_mode_string == "HALF_AWAY_FROM_ZERO") { + round_mode_ = ROUND_HALF_AWAY_FROM_ZERO; + } else if (round_mode_string == "HALF_TO_EVEN") { + OP_REQUIRES(ctx, mode_string == "SCALED", + errors::InvalidArgument("Round mode 'HALF_TO_EVEN' " + "only supported for mode 'SCALED', " + "but mode is '" + + mode_string + "'.")); + round_mode_ = ROUND_HALF_TO_EVEN; + } } void Compute(OpKernelContext* ctx) override { @@ -151,40 +185,37 @@ class QuantizeV2Op : public OpKernel { typename TTypes::Vec o = output->template flat(); static constexpr int num_bits = sizeof(T) * 8; const float max_abs = std::max(std::abs(min_range), std::abs(max_range)); - bool is_signed = std::is_signed::value; + const bool is_signed = std::is_signed::value; + float target_range; if (is_signed) { max_range = max_abs; min_range = -max_abs; // If it is signed, we try to keep 0.0 being 0 and drop one bucket. For // example, if it is 8 bits, we have the range [-127, 127]. So for input // range of [-x, x], the scale should be 254/(2*x). - const float target_range = - static_cast((uint64_t{1} << (num_bits - 1)) - 1); - const float scale_factor = target_range / max_abs; - // Note that std::round is used to round the number before the cast. - // std::round implements "round-half-away-zero", - // e.g., -5.5 gets rounded to -6, -5.4 goes to -5, 5.4 goes to 5, - // and 5.5 goes to 6. - o.device(ctx->template eigen_device()) = - (input.flat().cwiseMin(max_range).cwiseMax(min_range) * - scale_factor) - .round() - .template cast(); + target_range = static_cast((uint64_t{1} << (num_bits - 1)) - 1); } else { max_range = max_abs; min_range = 0.0; // If it is unsigned and num_bits == 8, the range with 8 bits is [0, // 255]. If the input range is [0, x], then the scale is x/255 instead // of 254 as in the case above. - const float target_range = - static_cast((uint64_t{1} << num_bits) - 1); - const float scale_factor = target_range / max_abs; - // Because input is unsigned, we don't need to implement "round away - // from zero". The fast path avoids unaryExpr. + target_range = static_cast((uint64_t{1} << num_bits) - 1); + } + const float scale_factor = target_range / max_abs; + if (round_mode_ == ROUND_HALF_TO_EVEN) { + // scalar_round_op_google implements "round-half-to-even". o.device(ctx->template eigen_device()) = (input.flat().cwiseMin(max_range).cwiseMax(min_range) * - scale_factor + - 0.5f) + scale_factor) + .unaryExpr(Eigen::internal::scalar_round_op_google()) + .template cast(); + } else if (round_mode_ == ROUND_HALF_AWAY_FROM_ZERO) { + // scalar_round_op implements "round-half-away-from-zero". + o.device(ctx->template eigen_device()) = + (input.flat().cwiseMin(max_range).cwiseMax(min_range) * + scale_factor) + .unaryExpr(Eigen::internal::scalar_round_op()) .template cast(); } } @@ -201,6 +232,7 @@ class QuantizeV2Op : public OpKernel { private: float half_range_; int mode_; + int round_mode_; }; REGISTER_KERNEL_BUILDER( diff --git a/tensorflow/core/kernels/quantize_op_test.cc b/tensorflow/core/kernels/quantize_op_test.cc index 8a370966b4..d2cc55a94d 100644 --- a/tensorflow/core/kernels/quantize_op_test.cc +++ b/tensorflow/core/kernels/quantize_op_test.cc @@ -82,6 +82,7 @@ TEST_F(QuantizedOpTest, QuantizeV2Quint8Scaled) { test::FillValues(&expected_output_max, {255.0}); test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); } + TEST_F(QuantizedOpTest, QuantizeV2Quint8ScaledSmallInputRange) { TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") .Input(FakeInput(DT_FLOAT)) @@ -170,6 +171,66 @@ TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledSmallInputRange) { test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); } +TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledRoundToEven) { + TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("T", DataTypeToEnum::v()) + .Attr("mode", "SCALED") + .Attr("round_mode", "HALF_TO_EVEN") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({7}), + {-126.5, 0.0, 1.0, 2.5, 3.5, 64.0, 127.0}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_QINT8, TensorShape({7})); + // Input element 0.0 should map to 0. + // Input element 127.0 maps to 127. + test::FillValues(&expected, {-126, 0, 1, 2, 4, 64, 127}); + test::ExpectTensorEqual(expected, *GetOutput(0)); + + Tensor expected_output_min(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_min, {-127.0}); + test::ExpectTensorEqual(expected_output_min, *GetOutput(1)); + + Tensor expected_output_max(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_max, {127.0}); + test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); +} + +TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledRoundAwayFromZero) { + TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("T", DataTypeToEnum::v()) + .Attr("mode", "SCALED") + .Attr("round_mode", "HALF_AWAY_FROM_ZERO") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({7}), + {-126.5, 0.0, 1.0, 2.5, 3.5, 64.0, 127.0}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_QINT8, TensorShape({7})); + // Input element 0.0 should map to 0. + // Input element 127.0 maps to 127. + test::FillValues(&expected, {-127, 0, 1, 3, 4, 64, 127}); + test::ExpectTensorEqual(expected, *GetOutput(0)); + + Tensor expected_output_min(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_min, {-127.0}); + test::ExpectTensorEqual(expected_output_min, *GetOutput(1)); + + Tensor expected_output_max(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_max, {127.0}); + test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); +} + TEST_F(QuantizedOpTest, QuantizeV2_32Bit) { TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") .Input(FakeInput(DT_FLOAT)) diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index c5935141f8..f73bc716d5 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -4859,6 +4859,9 @@ REGISTER_OP("QuantizeV2") .Output("output_max: float") .Attr("T: quantizedtype") .Attr("mode: {'MIN_COMBINED', 'MIN_FIRST', 'SCALED'} = 'MIN_COMBINED'") + .Attr( + "round_mode: {'HALF_AWAY_FROM_ZERO', 'HALF_TO_EVEN'} = " + "'HALF_AWAY_FROM_ZERO'") .SetShapeFn([](InferenceContext* c) { TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c)); ShapeHandle unused; @@ -4873,7 +4876,9 @@ Quantize the 'input' tensor of type float to 'output' tensor of type 'T'. [min_range, max_range] are scalar floats that specify the range for the 'input' data. The 'mode' attribute controls exactly which calculations are -used to convert the float values to their quantized equivalents. +used to convert the float values to their quantized equivalents. The +'round_mode' attribute controls which rounding tie-breaking algorithm is used +when rounding float values to their quantized equivalents. In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: @@ -4950,7 +4955,7 @@ From this we compute our scaling factor, s: Now we can quantize the elements of our tensor: ```c++ -result = (input * s).round_to_nearest() +result = round(input * s) ``` One thing to watch out for is that the operator may choose to adjust the diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index c00efb16ba..97dc63ebb1 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -2526,3 +2526,24 @@ def gather(params, indices, validate_indices=None, name=None, axis=0): gather.__doc__ = gen_array_ops.gather_v2.__doc__ + + +# Define quantize_v2 here in order to make name the second-to-last attribute, +# because round_mode was added later. +def quantize_v2(input, + min_range, + max_range, + T, + mode="MIN_COMBINED", + name=None, + round_mode="HALF_AWAY_FROM_ZERO"): + return gen_array_ops.quantize_v2(input, + min_range, + max_range, + T=T, + mode=mode, + name=name, + round_mode=round_mode) + + +quantize_v2.__doc__ = gen_array_ops.quantize_v2.__doc__ diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index d56a59de72..1c6f3cc534 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -1482,7 +1482,7 @@ tf_module { } member_method { name: "quantize_v2" - argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\'], " + argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], " } member_method { name: "quantized_concat" -- GitLab From 9e3c3f186812c780edb4eee5b1e154b7e2f6bf23 Mon Sep 17 00:00:00 2001 From: Sergio Guadarrama Date: Wed, 25 Oct 2017 14:21:52 -0700 Subject: [PATCH 368/573] Expose the name of the name_scope as a property. Create tf.get_name_scope() helper function. PiperOrigin-RevId: 173449188 --- tensorflow/python/framework/ops.py | 22 +++++++++++++++++++ .../tensorflow.keras.backend.name_scope.pbtxt | 4 ++++ .../api/golden/tensorflow.name_scope.pbtxt | 4 ++++ 3 files changed, 30 insertions(+) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 94c29c89df..61a5a4fcae 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -4694,6 +4694,24 @@ def get_default_graph(): return _default_graph_stack.get_default() +def get_name_scope(): + """Returns the current name scope in the default_graph. + + For example: + + ```python + with tf.name_scope('scope1'): + with tf.name_scope('scope2'): + print(tf.get_name_scope()) + ``` + would print the string `scope1/scope2`. + + Returns: + A string representing the current name scope. + """ + return get_default_graph().get_name_scope() + + def _assert_same_graph(original_item, item): """Fail if the 2 items are from different graphs. @@ -5019,6 +5037,10 @@ class name_scope(object): # pylint: disable=invalid-name ``` """ + @property + def name(self): + return self._name + def __init__(self, name, default_name=None, values=None): """Initialize the context manager. diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt index 43692a6c73..a2b98b1c27 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt @@ -2,6 +2,10 @@ path: "tensorflow.keras.backend.name_scope" tf_class { is_instance: "" is_instance: "" + member { + name: "name" + mtype: "" + } member_method { name: "__init__" argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt index 107f066c29..8041897013 100644 --- a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt @@ -2,6 +2,10 @@ path: "tensorflow.name_scope" tf_class { is_instance: "" is_instance: "" + member { + name: "name" + mtype: "" + } member_method { name: "__init__" argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " -- GitLab From 83b39b0753026c5617d286929129e3529a41ce9e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:36:18 -0700 Subject: [PATCH 369/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173451205 --- .../core/ops/compat/ops_history.v1.pbtxt | 67 +++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 15 ++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index e6aeb35e02..cec75f6799 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -22194,6 +22194,73 @@ op { } } } +op { + name: "QuantizeV2" + input_arg { + name: "input" + type: DT_FLOAT + } + input_arg { + name: "min_range" + type: DT_FLOAT + } + input_arg { + name: "max_range" + type: DT_FLOAT + } + output_arg { + name: "output" + type_attr: "T" + } + output_arg { + name: "output_min" + type: DT_FLOAT + } + output_arg { + name: "output_max" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT16 + type: DT_QUINT16 + type: DT_QINT32 + } + } + } + attr { + name: "mode" + type: "string" + default_value { + s: "MIN_COMBINED" + } + allowed_values { + list { + s: "MIN_COMBINED" + s: "MIN_FIRST" + s: "SCALED" + } + } + } + attr { + name: "round_mode" + type: "string" + default_value { + s: "HALF_AWAY_FROM_ZERO" + } + allowed_values { + list { + s: "HALF_AWAY_FROM_ZERO" + s: "HALF_TO_EVEN" + } + } + } +} op { name: "QuantizedAdd" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index a6886e465d..78f0fda408 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -17435,8 +17435,21 @@ op { } } } + attr { + name: "round_mode" + type: "string" + default_value { + s: "HALF_AWAY_FROM_ZERO" + } + allowed_values { + list { + s: "HALF_AWAY_FROM_ZERO" + s: "HALF_TO_EVEN" + } + } + } summary: "Quantize the \'input\' tensor of type float to \'output\' tensor of type \'T\'." - description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n numeric_limits::min()\nquantized = max(quantized, numeric_limits::min())\nquantized = min(quantized, numeric_limits::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = (input * s).round_to_nearest()\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them." + description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents. The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n numeric_limits::min()\nquantized = max(quantized, numeric_limits::min())\nquantized = min(quantized, numeric_limits::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them." } op { name: "QuantizedAdd" -- GitLab From 22298967f352c5d54856b22b69b2a6486b01f23c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:41:23 -0700 Subject: [PATCH 370/573] Adding learning rate decays found in Neural Optimizer Search with Reinforcement Learning [Bello et al, ICML2017] Also adding cosine decay. PiperOrigin-RevId: 173451903 --- .../python/training/learning_rate_decay.py | 227 +++++++++++++++++- .../training/learning_rate_decay_test.py | 93 +++++++ tensorflow/python/training/training.py | 3 + .../tools/api/golden/tensorflow.train.pbtxt | 12 + 4 files changed, 334 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py index e4a7964aaf..bb7762c8c5 100644 --- a/tensorflow/python/training/learning_rate_decay.py +++ b/tensorflow/python/training/learning_rate_decay.py @@ -18,12 +18,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import math + from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops - +from tensorflow.python.ops import random_ops def exponential_decay(learning_rate, global_step, decay_steps, decay_rate, staircase=False, name=None): @@ -412,3 +414,226 @@ def inverse_time_decay(learning_rate, global_step, decay_steps, decay_rate, const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) denom = math_ops.add(const, math_ops.multiply(decay_rate, p)) return math_ops.div(learning_rate, denom, name=name) + + +def cosine_decay(learning_rate, global_step, decay_steps, name=None): + """Applies cosine decay to the learning rate. + + See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent + with Warm Restarts. + + When training a model, it is often recommended to lower the learning rate as + the training progresses. This function applies a cosine decay function + to a provided initial learning rate. It requires a `global_step` value to + compute the decayed learning rate. You can just pass a TensorFlow variable + that you increment at each training step. + + The function returns the decayed learning rate. It is computed as: + ```python + global_step = min(global_step, decay_steps) + decayed = 0.5 * (1 + cos(pi * global_step / decay_steps)) + decayed_learning_rate = learning_rate * decayed + ``` + + Example usage: + ```python + decay_steps = 1000 + lr_decayed = cosine_decay(learning_rate, global_step, decay_steps) + ``` + + Args: + learning_rate: A scalar `float32` or `float64` Tensor or a Python number. + The initial learning rate. + global_step: A scalar `int32` or `int64` `Tensor` or a Python number. + Global step to use for the decay computation. + decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. + Number of steps to decay over. + name: String. Optional name of the operation. Defaults to 'CosineDecay'. + Returns: + A scalar `Tensor` of the same type as `learning_rate`. The decayed + learning rate. + Raises: + ValueError: if `global_step` is not supplied. + """ + if global_step is None: + raise ValueError("cosine decay requires global_step") + with ops.name_scope(name, "CosineDecay", + [learning_rate, global_step]) as name: + learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") + dtype = learning_rate.dtype + global_step = math_ops.cast(global_step, dtype) + decay_steps = math_ops.cast(decay_steps, dtype) + global_step = math_ops.minimum(global_step, decay_steps) + completed_fraction = global_step / decay_steps + cosine_decayed = 0.5 * ( + 1.0 + math_ops.cos(constant_op.constant(math.pi) * completed_fraction)) + + return math_ops.multiply(learning_rate, cosine_decayed) + + +def linear_cosine_decay(learning_rate, global_step, decay_steps, + num_periods=0.5, alpha=0.0, beta=0.001, + name=None): + """Applies linear cosine decay to the learning rate. + + See [Bello et al., ICML2017] Neural Optimizer Search with RL. + https://arxiv.org/abs/1709.07417 + + Note that linear cosine decay is more aggressive than cosine decay and + larger initial learning rates can typically be used. + + When training a model, it is often recommended to lower the learning rate as + the training progresses. This function applies a linear cosine decay function + to a provided initial learning rate. It requires a `global_step` value to + compute the decayed learning rate. You can just pass a TensorFlow variable + that you increment at each training step. + + The function returns the decayed learning rate. It is computed as: + ```python + global_step = min(global_step, decay_steps) + linear_decay = (decay_steps - global_step) / decay_steps) + cosine_decay = 0.5 * ( + 1 + cos(pi * 2 * num_periods * global_step / decay_steps)) + decayed = (alpha + linear_decay) * cosine_decay + beta + decayed_learning_rate = learning_rate * decayed + ``` + + Example usage: + ```python + decay_steps = 1000 + lr_decayed = linear_cosine_decay(learning_rate, global_step, decay_steps) + ``` + + Args: + learning_rate: A scalar `float32` or `float64` Tensor or a Python number. + The initial learning rate. + global_step: A scalar `int32` or `int64` `Tensor` or a Python number. + Global step to use for the decay computation. + decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. + Number of steps to decay over. + num_periods: Number of periods in the cosine part of the decay. + See computation above. + alpha: See computation above. + beta: See computation above. + name: String. Optional name of the operation. Defaults to + 'LinearCosineDecay'. + Returns: + A scalar `Tensor` of the same type as `learning_rate`. The decayed + learning rate. + Raises: + ValueError: if `global_step` is not supplied. + """ + if global_step is None: + raise ValueError("linear cosine decay requires global_step") + with ops.name_scope(name, "LinearCosineDecay", + [learning_rate, global_step]) as name: + learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") + dtype = learning_rate.dtype + global_step = math_ops.cast(global_step, dtype) + decay_steps = math_ops.cast(decay_steps, dtype) + num_periods = math_ops.cast(num_periods, dtype) + global_step = math_ops.minimum(global_step, decay_steps) + alpha = math_ops.cast(alpha, dtype) + beta = math_ops.cast(beta, dtype) + + linear_decayed = (decay_steps - global_step) / decay_steps + completed_fraction = global_step / decay_steps + fraction = 2.0 * num_periods * completed_fraction + cosine_decayed = 0.5 * ( + 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) + + linear_cosine_decayed = (alpha + linear_decayed) * cosine_decayed + beta + return math_ops.multiply(learning_rate, linear_cosine_decayed, name=name) + + +def noisy_linear_cosine_decay(learning_rate, global_step, decay_steps, + initial_variance=1.0, variance_decay=0.55, + num_periods=0.5, alpha=0.0, beta=0.001, + name=None): + """Applies noisy linear cosine decay to the learning rate. + + See [Bello et al., ICML2017] Neural Optimizer Search with RL. + https://arxiv.org/abs/1709.07417 + + Note that linear cosine decay is more aggressive than cosine decay and + larger initial learning rates can typically be used. + + When training a model, it is often recommended to lower the learning rate as + the training progresses. This function applies a noisy linear + cosine decay function to a provided initial learning rate. + It requires a `global_step` value to compute the decayed learning rate. + You can just pass a TensorFlow variable that you increment at each + training step. + + The function returns the decayed learning rate. It is computed as: + ```python + global_step = min(global_step, decay_steps) + linear_decay = (decay_steps - global_step) / decay_steps) + cosine_decay = 0.5 * ( + 1 + cos(pi * 2 * num_periods * global_step / decay_steps)) + decayed = (alpha + linear_decay + eps_t) * cosine_decay + beta + decayed_learning_rate = learning_rate * decayed + ``` + where eps_t is 0-centered gaussian noise with variance + initial_variance / (1 + global_step) ** variance_decay + + Example usage: + ```python + decay_steps = 1000 + lr_decayed = noisy_linear_cosine_decay( + learning_rate, global_step, decay_steps) + ``` + + Args: + learning_rate: A scalar `float32` or `float64` Tensor or a Python number. + The initial learning rate. + global_step: A scalar `int32` or `int64` `Tensor` or a Python number. + Global step to use for the decay computation. + decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. + Number of steps to decay over. + initial_variance: initial variance for the noise. See computation above. + variance_decay: decay for the noise's variance. See computation above. + num_periods: Number of periods in the cosine part of the decay. + See computation above. + alpha: See computation above. + beta: See computation above. + name: String. Optional name of the operation. Defaults to + 'NoisyLinearCosineDecay'. + Returns: + A scalar `Tensor` of the same type as `learning_rate`. The decayed + learning rate. + Raises: + ValueError: if `global_step` is not supplied. + """ + if global_step is None: + raise ValueError("noisy linear cosine decay requires global_step") + with ops.name_scope(name, "NoisyLinearCosineDecay", + [learning_rate, global_step]) as name: + learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") + dtype = learning_rate.dtype + global_step = math_ops.cast(global_step, dtype) + decay_steps = math_ops.cast(decay_steps, dtype) + global_step = math_ops.minimum(global_step, decay_steps) + initial_variance = math_ops.cast(initial_variance, dtype) + variance_decay = math_ops.cast(variance_decay, dtype) + num_periods = math_ops.cast(num_periods, dtype) + alpha = math_ops.cast(alpha, dtype) + beta = math_ops.cast(beta, dtype) + + linear_decayed = (decay_steps - global_step) / decay_steps + variance = initial_variance / ( + math_ops.pow(1.0 + global_step, variance_decay)) + std = math_ops.sqrt(variance) + noisy_linear_decayed = ( + linear_decayed + random_ops.random_normal( + linear_decayed.shape, stddev=std)) + + completed_fraction = global_step / decay_steps + fraction = 2.0 * num_periods * completed_fraction + cosine_decayed = 0.5 * ( + 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) + noisy_linear_cosine_decayed = ( + (alpha + noisy_linear_decayed) * cosine_decayed + beta) + + return math_ops.multiply( + learning_rate, noisy_linear_cosine_decayed, name=name) diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py index 77da3099fe..34c300eae7 100644 --- a/tensorflow/python/training/learning_rate_decay_test.py +++ b/tensorflow/python/training/learning_rate_decay_test.py @@ -340,5 +340,98 @@ class InverseDecayTest(test_util.TensorFlowTestCase): increment_step.op.run() +class CosineDecayTest(test_util.TensorFlowTestCase): + + def np_cosine_decay(self, step, decay_steps): + step = min(step, decay_steps) + completed_fraction = step / decay_steps + return 0.5 * (1.0 + math.cos(math.pi * completed_fraction)) + + def testDecay(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + decayed_lr = learning_rate_decay.cosine_decay( + initial_lr, step, num_training_steps) + expected = self.np_cosine_decay(step, num_training_steps) + self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + + +class LinearCosineDecayTest(test_util.TensorFlowTestCase): + + def np_linear_cosine_decay(self, + step, + decay_steps, + alpha=0.0, + beta=0.001, + num_periods=0.5): + step = min(step, decay_steps) + linear_decayed = float(decay_steps - step) / decay_steps + fraction = 2.0 * num_periods * step / float(decay_steps) + cosine_decayed = 0.5 * (1.0 + math.cos(math.pi * fraction)) + return (alpha + linear_decayed) * cosine_decayed + beta + + def testDefaultDecay(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + decayed_lr = learning_rate_decay.linear_cosine_decay( + initial_lr, step, num_training_steps) + expected = self.np_linear_cosine_decay(step, num_training_steps) + self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + + def testNonDefaultDecay(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + decayed_lr = learning_rate_decay.linear_cosine_decay( + initial_lr, + step, + num_training_steps, + alpha=0.1, + beta=1e-4, + num_periods=5) + expected = self.np_linear_cosine_decay( + step, + num_training_steps, + alpha=0.1, + beta=1e-4, + num_periods=5) + self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + + +class NoisyLinearCosineDecayTest(test_util.TensorFlowTestCase): + + def testDefaultNoisyLinearCosine(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + # No numerical check because of noise + decayed_lr = learning_rate_decay.noisy_linear_cosine_decay( + initial_lr, step, num_training_steps) + decayed_lr.eval() + + def testNonDefaultNoisyLinearCosine(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + # No numerical check because of noise + decayed_lr = learning_rate_decay.noisy_linear_cosine_decay( + initial_lr, + step, + num_training_steps, + initial_variance=0.5, + variance_decay=0.1, + alpha=0.1, + beta=1e-4, + num_periods=5) + decayed_lr.eval() + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/training/training.py b/tensorflow/python/training/training.py index 741dddc991..fa02ad84cc 100644 --- a/tensorflow/python/training/training.py +++ b/tensorflow/python/training/training.py @@ -37,6 +37,9 @@ See the @{$python/train} guide. @@clip_by_average_norm @@clip_by_global_norm @@global_norm +@@cosine_decay +@@linear_cosine_decay +@@noisy_linear_cosine_decay @@exponential_decay @@inverse_time_decay @@natural_exp_decay diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt index edc29e62dd..e73f6f6e63 100644 --- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt @@ -264,6 +264,10 @@ tf_module { name: "checkpoint_exists" argspec: "args=[\'checkpoint_prefix\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cosine_decay" + argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "create_global_step" argspec: "args=[\'graph\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -328,6 +332,10 @@ tf_module { name: "limit_epochs" argspec: "args=[\'tensor\', \'num_epochs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } + member_method { + name: "linear_cosine_decay" + argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'0.0\', \'0.001\', \'None\'], " + } member_method { name: "list_variables" argspec: "args=[\'ckpt_dir_or_file\'], varargs=None, keywords=None, defaults=None" @@ -364,6 +372,10 @@ tf_module { name: "natural_exp_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "noisy_linear_cosine_decay" + argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], " + } member_method { name: "piecewise_constant" argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " -- GitLab From 77d8f9ec51fa6daded8193841373d360fd978f1a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 14:44:00 -0700 Subject: [PATCH 371/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173452304 --- tensorflow/go/op/wrappers.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 958ce6d040..cc8165e2c7 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -13789,11 +13789,21 @@ func QuantizeV2Mode(value string) QuantizeV2Attr { } } +// QuantizeV2RoundMode sets the optional round_mode attribute to value. +// If not specified, defaults to "HALF_AWAY_FROM_ZERO" +func QuantizeV2RoundMode(value string) QuantizeV2Attr { + return func(m optionalAttr) { + m["round_mode"] = value + } +} + // Quantize the 'input' tensor of type float to 'output' tensor of type 'T'. // // [min_range, max_range] are scalar floats that specify the range for // the 'input' data. The 'mode' attribute controls exactly which calculations are -// used to convert the float values to their quantized equivalents. +// used to convert the float values to their quantized equivalents. The +// 'round_mode' attribute controls which rounding tie-breaking algorithm is used +// when rounding float values to their quantized equivalents. // // In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: // @@ -13870,7 +13880,7 @@ func QuantizeV2Mode(value string) QuantizeV2Attr { // // Now we can quantize the elements of our tensor: // ```c++ -// result = (input * s).round_to_nearest() +// result = round(input * s) // ``` // // One thing to watch out for is that the operator may choose to adjust the -- GitLab From 71beb2dbf8eb41ac03ea4cd52a771ad31b125e7e Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Wed, 25 Oct 2017 15:00:21 -0700 Subject: [PATCH 372/573] Remove unnecessary -ldl We no longer use dlopen() in :simple_orc_jit (after cl/173277862), so the -ldl flag is no longer needed. PiperOrigin-RevId: 173454761 --- tensorflow/compiler/xla/service/cpu/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 56bc1a6706..ef8eed3f88 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -146,7 +146,6 @@ cc_library( name = "simple_orc_jit", srcs = ["simple_orc_jit.cc"], hdrs = ["simple_orc_jit.h"], - linkopts = ["-ldl"], deps = [ ":compiler_functor", ":cpu_runtime", -- GitLab From 5fe90b57748714341d02b2b44a7ec8ff27123bc0 Mon Sep 17 00:00:00 2001 From: Yangzihao Wang Date: Wed, 25 Oct 2017 15:10:29 -0700 Subject: [PATCH 373/573] Force the CUDA runtime initialization before device creation. This is to avoid silent failure and garbage results produced when launching two TensorFlow programs simultaneously in two different processes. PiperOrigin-RevId: 173456597 --- .../core/common_runtime/gpu/gpu_device.cc | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc index 12d44cc6b7..2c906ed220 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_device.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc @@ -652,6 +652,34 @@ Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options, if (static_cast(n) > valid_gpu_ids.size()) { n = valid_gpu_ids.size(); } + // Save the original device. + int original_device = 0; + cudaError_t err = cudaGetDevice(&original_device); + if (err != cudaSuccess) { + return errors::Internal("cudaGetDevice() failed. Status: ", + cudaGetErrorString(err)); + } + // Force to implicitly initialize CUDA runtime on each valid GPU before + // CreateGPUDevice(). + for (int gpu_id : valid_gpu_ids) { + err = cudaSetDevice(gpu_id); + if (err != cudaSuccess) { + return errors::Internal("cudaSetDevice() on GPU:", gpu_id, + " failed. Status: ", cudaGetErrorString(err)); + } + err = cudaFree(nullptr); + if (err != cudaSuccess) { + return errors::Internal( + "CUDA runtime implicit initialization on GPU:", gpu_id, + " failed. Status: ", cudaGetErrorString(err)); + } + } + // Reset to the original device. + err = cudaSetDevice(original_device); + if (err != cudaSuccess) { + return errors::Internal("cudaSetDevice() on GPU:", original_device, + " failed. Status: ", cudaGetErrorString(err)); + } for (int i = 0; i < n; i++) { BaseGPUDevice* gpu_device; TF_RETURN_IF_ERROR(CreateGPUDevice( -- GitLab From df299e1a0c91f50acf4868c7bb3e0ea93b52db7b Mon Sep 17 00:00:00 2001 From: Suharsh Sivakumar Date: Wed, 25 Oct 2017 15:10:45 -0700 Subject: [PATCH 374/573] Allow device specification for variables added by quantize graph rewriter. PiperOrigin-RevId: 173456646 --- tensorflow/contrib/quantize/BUILD | 4 + .../contrib/quantize/python/quantize_graph.py | 46 +++++--- .../quantize/python/quantize_graph_test.py | 102 ++++++++++++++---- .../contrib/quantize/python/quantize_test.py | 23 ---- 4 files changed, 119 insertions(+), 56 deletions(-) diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD index 0d6c71965c..2c0ffaf6c0 100644 --- a/tensorflow/contrib/quantize/BUILD +++ b/tensorflow/contrib/quantize/BUILD @@ -219,9 +219,13 @@ py_test( srcs_version = "PY2AND3", deps = [ ":quantize_graph", + "//tensorflow/contrib/layers:layers_py", + "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:init_ops", + "//tensorflow/python:nn_ops", "//tensorflow/python:platform_test", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py index aaf3e92b8e..d647bb94e8 100644 --- a/tensorflow/contrib/quantize/python/quantize_graph.py +++ b/tensorflow/contrib/quantize/python/quantize_graph.py @@ -25,7 +25,10 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import variables -def _create_graph(input_graph, is_training, elements=None): +def _create_graph(input_graph, + is_training, + elements=None, + device_name_or_function=None): """Returns a transformed training input_graph for simulated quantization. The forward pass has fake quantization ops inserted to simulate the error @@ -36,12 +39,12 @@ def _create_graph(input_graph, is_training, elements=None): is_training: Whether quantizing training or eval graph. elements: (Optional) List of Tensors and Operations in input_graph whose corresponding elements in the new graph will be returned. + device_name_or_function: (Optional) The device name or function to use. Returns: - Returns a tuple(g, l) where: g is new tf.Graph that is rewritten for simulated quantization. l is a list of Tensors/Operations in g corresponding to the provided input - elements. + elements, if elements is not None. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or @@ -49,11 +52,14 @@ def _create_graph(input_graph, is_training, elements=None): """ # TODO(suharshs): Describe the process in more detail in the doc string. g = copy_graph.CopyGraph(input_graph) - fold_batch_norms.FoldBatchNorms(g) - quantize.Quantize(g, is_training=is_training) - return_elements = [] + with g.as_default(): + with ops.device(device_name_or_function): + fold_batch_norms.FoldBatchNorms(g) + quantize.Quantize(g, is_training=is_training) if elements is None: - elements = [] + return g + + return_elements = [] for element in elements: if isinstance(element, (ops.Tensor, variables.Variable)): return_elements.append(g.get_tensor_by_name(element.name)) @@ -66,7 +72,9 @@ def _create_graph(input_graph, is_training, elements=None): return g, return_elements -def create_training_graph(input_graph, elements=None): +def create_training_graph(input_graph, + elements=None, + device_name_or_function=None): """Returns a transformed training input_graph for simulated quantization. The forward pass has fake quantization ops inserted to simulate the error @@ -76,21 +84,25 @@ def create_training_graph(input_graph, elements=None): input_graph: The tf.Graph to be transformed. elements: (Optional) List of Tensors and Operations in input_graph whose corresponding elements in the new graph will be returned. + device_name_or_function: (Optional) The device name or function to use. Returns: - Returns a tuple(g, l) where: g is new tf.Graph that is rewritten for simulated quantization. l is a list of Tensors/Operations in g corresponding to the provided input - elements. + elements, if elements is not None. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or tf.Operation. """ - return _create_graph(input_graph, True, elements) + return _create_graph( + input_graph=input_graph, + is_training=True, + elements=elements, + device_name_or_function=device_name_or_function) -def create_eval_graph(input_graph, elements=None): +def create_eval_graph(input_graph, elements=None, device_name_or_function=None): """Returns a transformed eval input_graph for simulated quantization. The forward pass has fake quantization ops inserted to simulate the error @@ -100,15 +112,19 @@ def create_eval_graph(input_graph, elements=None): input_graph: The tf.Graph to be transformed. elements: (Optional) List of Tensors and Operations in input_graph whose corresponding elements in the new graph will be returned. + device_name_or_function: (Optional) The device name or function to use. Returns: - Returns a tuple(g, l) where: g is new tf.Graph that is rewritten for simulated quantization. l is a list of Tensors/Operations in g corresponding to the provided input - elements. + elements, if elements is not None. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or tf.Operation. """ - return _create_graph(input_graph, False, elements) + return _create_graph( + input_graph=input_graph, + is_training=False, + elements=elements, + device_name_or_function=device_name_or_function) diff --git a/tensorflow/contrib/quantize/python/quantize_graph_test.py b/tensorflow/contrib/quantize/python/quantize_graph_test.py index 382076672a..3407ace391 100644 --- a/tensorflow/contrib/quantize/python/quantize_graph_test.py +++ b/tensorflow/contrib/quantize/python/quantize_graph_test.py @@ -18,29 +18,41 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.contrib.layers.python.layers import layers from tensorflow.contrib.quantize.python import quantize_graph from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import nn_ops from tensorflow.python.ops import variables from tensorflow.python.platform import googletest -class QuantizeTest(test_util.TensorFlowTestCase): +class QuantizeGraphTest(test_util.TensorFlowTestCase): # We have a lot of other tests that test the details of the rewrite, here we # just the specific features of the quantize_graph API. def testReturnedElementsTraining(self): + self._TestReturnElements(True) + + def testReturnedElementsEval(self): + self._TestReturnElements(False) + + def _TestReturnElements(self, is_training): graph = ops.Graph() with graph.as_default(): a = constant_op.constant(1.0) b = variables.Variable(2.0) c = a + b elements = [a, b, c.op] - for element in elements: - print(element) - q_graph, returned_elements = quantize_graph.create_training_graph( - graph, elements=elements) + if is_training: + q_graph, returned_elements = quantize_graph.create_training_graph( + graph, elements=elements) + else: + q_graph, returned_elements = quantize_graph.create_eval_graph( + graph, elements=elements) # Make sure q_graph is different from graph. self.assertTrue(graph != q_graph) # Check that the returned elements are part of the new graph. @@ -50,25 +62,79 @@ class QuantizeTest(test_util.TensorFlowTestCase): for element, returned_element in zip(elements, returned_elements): self.assertEqual(element.name, returned_element.name) - # We have a lot of other tests that test the details of the rewrite, here we - # just the specific features of the quantize_graph API. - def testReturnedElementsEval(self): + def testNoReturnElementsTraining(self): + self._TestNoReturnElements(True) + + def testNoReturnElementsEval(self): + self._TestNoReturnElements(False) + + def _TestNoReturnElements(self, is_training): graph = ops.Graph() with graph.as_default(): a = constant_op.constant(1.0) b = variables.Variable(2.0) - c = a + b - elements = [a, b, c.op] - q_graph, returned_elements = quantize_graph.create_eval_graph( - graph, elements=elements) + _ = a + b + if is_training: + q_graph = quantize_graph.create_training_graph(graph) + else: + q_graph = quantize_graph.create_eval_graph(graph) + # Check that quantize_graph didn't return a tuple when elements isn't + # provided. + self.assertTrue(isinstance(q_graph, ops.Graph)) # Make sure q_graph is different from graph. self.assertTrue(graph != q_graph) - # Check that the returned elements are part of the new graph. - for returned_element in returned_elements: - self.assertEqual(q_graph, returned_element.graph) - # Check that the elements match with the one from the input graph. - for element, returned_element in zip(elements, returned_elements): - self.assertEqual(element.name, returned_element.name) + + def testDeviceNameTraining(self): + self._TestDeviceName(True) + + def testDeviceNameEval(self): + self._TestDeviceName(False) + + def _TestDeviceName(self, is_training): + graph = ops.Graph() + with graph.as_default(): + batch_size, height, width, depth = 5, 128, 128, 3 + inputs = array_ops.zeros((batch_size, height, width, depth)) + conv = layers.conv2d( + inputs, + 32, [5, 5], + stride=2, + padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=None, + scope='test') + _ = nn_ops.relu6(conv) + + device_name = '/job:oink/task:0/device:CPU:0' + if is_training: + q_graph = quantize_graph.create_training_graph( + graph, device_name_or_function=device_name) + else: + q_graph = quantize_graph.create_eval_graph( + graph, device_name_or_function=device_name) + + orig_variable_names = set( + [v.name for v in graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + q_variables = q_graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + # Ensure that variables were added. + self.assertTrue(len(orig_variable_names) < len(q_variables)) + # All added variables should have the specified device name. + for var in q_variables: + if var.name not in orig_variable_names: + self.assertEqual(var.device, device_name) + + def _WeightInit(self, stddev): + """Returns truncated normal variable initializer. + + Function is defined purely to shorten the name so that it stops wrapping. + + Args: + stddev: Standard deviation of normal variable. + + Returns: + An initialized that initialzes with a truncated normal variable. + """ + return init_ops.truncated_normal_initializer(stddev=stddev) if __name__ == '__main__': diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py index a6bd809bb7..4a82eac197 100644 --- a/tensorflow/contrib/quantize/python/quantize_test.py +++ b/tensorflow/contrib/quantize/python/quantize_test.py @@ -65,28 +65,5 @@ class QuantizeTest(test_util.TensorFlowTestCase): """ return init_ops.truncated_normal_initializer(stddev=stddev) - def _AssertInputOpsAre(self, op, in_op_names): - """Asserts that all inputs to op come from in_op_names (disregarding order). - - Args: - op: Operation to check inputs for. - in_op_names: List of strings, operations where all op's inputs should - come from. - """ - expected_inputs = [in_op_name + ':0' for in_op_name in in_op_names] - self.assertItemsEqual([t.name for t in op.inputs], expected_inputs) - - def _AssertOutputGoesToOps(self, op, graph, out_op_names): - """Asserts that outputs from op go to out_op_names (and perhaps others). - - Args: - op: Operation to check outputs for. - graph: Graph where output operations are located. - out_op_names: List of strings, operations where op's outputs should go. - """ - for out_op_name in out_op_names: - out_op = graph.get_operation_by_name(out_op_name) - self.assertIn(op.outputs[0].name, [str(t.name) for t in out_op.inputs]) - if __name__ == '__main__': googletest.main() -- GitLab From 2ece96046d32c205c78393441404604db8706b9b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 23 Oct 2017 20:25:44 +0000 Subject: [PATCH 375/573] Add clang style check as part of the sanity check. This fix is an effort to add clang style check as part of the sanity check. In `CONTRIBUTING.md` it has been advised to run `clang-format --style=google file.cc` so that Google coding style is conformed. However, there is no sanity check in the current Jenkins build so current .cc and .h files in the repo are not really conforming to the coding style. This actually causes issues. In case a PR is submitted with `clang-format --style=google file.cc`, the reviewer may see additional unrelated changes which might be a distraction. The developer may also spent additional time to manually check for any discrepancies manually with additional unrelated style changes. This fix adds the clang-format check to the ci build so that when `ci_sanity.sh` is running, it will use clang-format to make sure the code is conforming to the coding style as specified in `CONTRIBUTING.md`. One thing that might need to take notice is the header order of the Eigen library. See https://github.com/tensorflow/tensorflow/pull/13907#issuecomment-338718110 for further details. Basically, if Eigen headers could be placed in any order, then no additional steps are needed. Otherwise, it is always possible to place the Eigen headers at the top, then leave one empty line like: ```cpp ``` In this way, even a run of `clang-format -i --style=google file.cc` will still respect the order and leave Eigen header at the top. This PR is experimeal so it only checks `tensorflow/core/ops` directory. Other files could be added if this PR is OK. This PR also sanitizes all files in `tensorflow/core/ops` directory so that it conforms to coding style requirement. Signed-off-by: Yong Tang --- tensorflow/tools/ci_build/ci_sanity.sh | 6 +++- .../tools/ci_build/clang_format_check.sh | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100755 tensorflow/tools/ci_build/clang_format_check.sh diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 26053de4e9..7cf97dacf0 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -426,6 +426,10 @@ do_code_link_check() { tensorflow/tools/ci_build/code_link_check.sh } +do_clang_format_check() { + CLANG_FORMAT=clang-format-3.8 tensorflow/tools/ci_build/clang_format_check.sh +} + do_check_load_py_test() { BUILD_CMD="bazel build ${BAZEL_FLAGS} //tensorflow/tools/pip_package:check_load_py_test" ${BUILD_CMD} @@ -439,7 +443,7 @@ do_check_load_py_test() { } # Supply all sanity step commands and descriptions -SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check") +SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_clang_format_check") SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links") INCREMENTAL_FLAG="" diff --git a/tensorflow/tools/ci_build/clang_format_check.sh b/tensorflow/tools/ci_build/clang_format_check.sh new file mode 100755 index 0000000000..5f252d7f35 --- /dev/null +++ b/tensorflow/tools/ci_build/clang_format_check.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# please run this at root directory of tensorflow +success=1 + +CLANG_FORMAT=${CLANG_FORMAT:-clang-format} + +# only tensorflow/core/ops is checked at the moment for experimental purpose +for filename in $(find tensorflow/core/ops -name *.h -o -name *.cc); do + $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null + if [ ! $? -eq 0 ]; then + success=0 + echo File $filename is not properly formatted with "clang-format --style=google" + fi +done + +if [ $success == 0 ]; then + echo Clang format check fails. + exit 1 +fi + +echo Clang format check success. -- GitLab From 44ce1cbe1bb153b93904ebb4c1b64d1079309065 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 23 Oct 2017 20:39:48 +0000 Subject: [PATCH 376/573] Add installation of clang-format-3.8, this will not conflict with clang Add installation of clang-format-3.8, this will not conflict with clang as command line needs to run with `clang-format-3.8 -i --style=Google` Signed-off-by: Yong Tang --- tensorflow/tools/ci_build/install/install_deb_packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh index da1f2199d0..03ca4716b4 100755 --- a/tensorflow/tools/ci_build/install/install_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh @@ -41,6 +41,7 @@ apt-get install -y --no-install-recommends \ autoconf \ automake \ build-essential \ + clang-format-3.8 \ curl \ ffmpeg \ git \ -- GitLab From 5c9e7dc293cbeda6b085238fabacba02560424b4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 23 Oct 2017 20:41:43 +0000 Subject: [PATCH 377/573] Sanitize files in tensorflow/core/ops with clang-format -i Signed-off-by: Yong Tang --- tensorflow/core/ops/bitwise_ops.cc | 20 ++- .../compat/backwards_compatibility_test.cc | 5 +- tensorflow/core/ops/data_flow_ops.cc | 3 - tensorflow/core/ops/image_ops.cc | 56 +++--- tensorflow/core/ops/linalg_ops.cc | 1 - tensorflow/core/ops/math_grad_test.cc | 17 +- tensorflow/core/ops/math_ops.cc | 168 +++++++++++++----- tensorflow/core/ops/nn_ops.cc | 12 +- tensorflow/core/ops/nn_ops_test.cc | 12 +- tensorflow/core/ops/sparse_ops_test.cc | 4 +- tensorflow/core/ops/stateless_random_ops.cc | 9 +- 11 files changed, 196 insertions(+), 111 deletions(-) diff --git a/tensorflow/core/ops/bitwise_ops.cc b/tensorflow/core/ops/bitwise_ops.cc index 3156162b78..2889953bdb 100644 --- a/tensorflow/core/ops/bitwise_ops.cc +++ b/tensorflow/core/ops/bitwise_ops.cc @@ -56,35 +56,45 @@ representation of that entry. 8- or 16-bit inputs and then aggregate the resulting counts. )doc"); -REGISTER_OP("BitwiseAnd").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("BitwiseAnd") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise AND of `x` and `y`. The result will have those bits set, that are set in both `x` and `y`. The computation is performed on the underlying representations of `x` and `y`. )doc"); -REGISTER_OP("BitwiseOr").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("BitwiseOr") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise OR of `x` and `y`. The result will have those bits set, that are set in `x`, `y` or both. The computation is performed on the underlying representations of `x` and `y`. )doc"); -REGISTER_OP("BitwiseXor").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("BitwiseXor") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise XOR of `x` and `y`. The result will have those bits set, that are different in `x` and `y`. The computation is performed on the underlying representations of `x` and `y`. )doc"); -REGISTER_OP("LeftShift").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("LeftShift") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise left-shift of `x` and `y`. If `y` is negative, or greater than or equal to the width of `x` in bits the result is implementation defined. )doc"); -REGISTER_OP("RightShift").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("RightShift") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise right-shift of `x` and `y`. Performs a logical shift for unsigned integer types, and an arithmetic shift diff --git a/tensorflow/core/ops/compat/backwards_compatibility_test.cc b/tensorflow/core/ops/compat/backwards_compatibility_test.cc index 6e05ae4be4..add05d6610 100644 --- a/tensorflow/core/ops/compat/backwards_compatibility_test.cc +++ b/tensorflow/core/ops/compat/backwards_compatibility_test.cc @@ -25,9 +25,8 @@ namespace tensorflow { namespace { TEST(BackwardsCompatibilityTest, IsCompatible) { - OpCompatibilityLib compatibility("tensorflow/core/ops", - strings::StrCat("v", TF_MAJOR_VERSION), - nullptr); + OpCompatibilityLib compatibility( + "tensorflow/core/ops", strings::StrCat("v", TF_MAJOR_VERSION), nullptr); Env* env = Env::Default(); int changed_ops = 0; diff --git a/tensorflow/core/ops/data_flow_ops.cc b/tensorflow/core/ops/data_flow_ops.cc index 8e24ea70cb..3b1ed217ce 100644 --- a/tensorflow/core/ops/data_flow_ops.cc +++ b/tensorflow/core/ops/data_flow_ops.cc @@ -2225,7 +2225,6 @@ this op will block until it does. This Op is optimized for performance. )doc"); - REGISTER_OP("StageSize") .Output("size: int32") .Attr("capacity: int >= 0 = 0") @@ -2354,7 +2353,6 @@ REGISTER_OP("MapIncompleteSize") Op returns the number of incomplete elements in the underlying container. )doc"); - REGISTER_OP("MapClear") .Attr("capacity: int >= 0 = 0") .Attr("memory_limit: int >= 0 = 0") @@ -2367,7 +2365,6 @@ REGISTER_OP("MapClear") Op removes all elements in the underlying container. )doc"); - // OrderedMap REGISTER_OP("OrderedMapStage") .Input("key: int64") diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc index e9bf29d172..c3f8006415 100644 --- a/tensorflow/core/ops/image_ops.cc +++ b/tensorflow/core/ops/image_ops.cc @@ -925,27 +925,27 @@ use_image_if_no_bounding_boxes: Controls behavior if no bounding boxes supplied. )doc"); REGISTER_OP("SampleDistortedBoundingBoxV2") - .Input("image_size: T") - .Input("bounding_boxes: float") - .Input("min_object_covered: float") - .Output("begin: T") - .Output("size: T") - .Output("bboxes: float") - .Attr("T: {uint8, int8, int16, int32, int64}") - .Attr("seed: int = 0") - .Attr("seed2: int = 0") - .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]") - .Attr("area_range: list(float) = [0.05, 1.0]") - .Attr("max_attempts: int = 100") - .Attr("use_image_if_no_bounding_boxes: bool = false") - .SetIsStateful() - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Vector(3)); - c->set_output(1, c->Vector(3)); - c->set_output(2, c->MakeShape({1, 1, 4})); - return Status::OK(); - }) - .Doc(R"doc( + .Input("image_size: T") + .Input("bounding_boxes: float") + .Input("min_object_covered: float") + .Output("begin: T") + .Output("size: T") + .Output("bboxes: float") + .Attr("T: {uint8, int8, int16, int32, int64}") + .Attr("seed: int = 0") + .Attr("seed2: int = 0") + .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]") + .Attr("area_range: list(float) = [0.05, 1.0]") + .Attr("max_attempts: int = 100") + .Attr("use_image_if_no_bounding_boxes: bool = false") + .SetIsStateful() + .SetShapeFn([](InferenceContext* c) { + c->set_output(0, c->Vector(3)); + c->set_output(1, c->Vector(3)); + c->set_output(2, c->MakeShape({1, 1, 4})); + return Status::OK(); + }) + .Doc(R"doc( Generate a single randomly distorted bounding box for an image. Bounding box annotations are often supplied in addition to ground-truth labels @@ -1236,16 +1236,16 @@ method: A string specifying the interpolation method. Only 'bilinear' is // -------------------------------------------------------------------------- REGISTER_OP("NonMaxSuppression") - .Input("boxes: float") - .Input("scores: float") - .Input("max_output_size: int32") - .Output("selected_indices: int32") - .Attr("iou_threshold: float = 0.5") - .SetShapeFn([](InferenceContext* c) { + .Input("boxes: float") + .Input("scores: float") + .Input("max_output_size: int32") + .Output("selected_indices: int32") + .Attr("iou_threshold: float = 0.5") + .SetShapeFn([](InferenceContext* c) { c->set_output(0, c->Vector(c->UnknownDim())); return Status::OK(); }) - .Doc(R"doc( + .Doc(R"doc( Greedily selects a subset of bounding boxes in descending order of score, pruning away boxes that have high intersection-over-union (IOU) overlap with previously selected boxes. Bounding boxes are supplied as diff --git a/tensorflow/core/ops/linalg_ops.cc b/tensorflow/core/ops/linalg_ops.cc index 76e2149522..4851619f83 100644 --- a/tensorflow/core/ops/linalg_ops.cc +++ b/tensorflow/core/ops/linalg_ops.cc @@ -25,7 +25,6 @@ using shape_inference::ShapeHandle; namespace { - // Return in the result of making the end of a square matrix. Status MakeBatchSquareMatrix(InferenceContext* c, ShapeHandle input, ShapeHandle* out) { diff --git a/tensorflow/core/ops/math_grad_test.cc b/tensorflow/core/ops/math_grad_test.cc index 2b4b35547b..8dcd3e815f 100644 --- a/tensorflow/core/ops/math_grad_test.cc +++ b/tensorflow/core/ops/math_grad_test.cc @@ -385,7 +385,7 @@ class TestOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_CPU), TestOp); #ifdef TENSORFLOW_USE_SYCL REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_SYCL), TestOp); -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TEST_F(MathGradTest, Error_Reporting) { auto x = test::AsTensor({-3.f}); @@ -557,11 +557,10 @@ TEST_F(MathGradTest, Acosh) { TEST_F(MathGradTest, Atanh) { auto x = test::AsTensor({-0.3f, -0.2f, -0.1f, 0.1f, 0.2f, 0.3f}, TensorShape({2, 3})); - auto g = [](float x) { - return 1.f / (1.f - x * x); - }; + auto g = [](float x) { return 1.f / (1.f - x * x); }; auto dx = test::AsTensor( - {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)}, TensorShape({2, 3})); + {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)}, + TensorShape({2, 3})); auto ans = SymGrad("Atanh", x); test::ExpectClose(ans, dx); } @@ -761,7 +760,7 @@ TEST_F(MathGradTest, Pow) { } } -//TODO{lukeiwanski}: Implement Complex Pow for SYCL +// TODO{lukeiwanski}: Implement Complex Pow for SYCL #ifndef TENSORFLOW_USE_SYCL TEST_F(MathGradTest, ComplexPow) { auto x = test::AsTensor({0.f, 2.f, -2.f}, TensorShape({3})); @@ -781,7 +780,7 @@ TEST_F(MathGradTest, ComplexPow) { dy, test::AsTensor({h(0.f, 2.f), h(2.f, 2.f), h(-2.f, 2.f)}, TensorShape({3}))); } -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TEST_F(MathGradTest, Maximum) { auto x = test::AsTensor({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, @@ -943,7 +942,7 @@ TEST_F(MathGradTest, MatMul_11) { test::ExpectClose(dy, MatMul(dz, true, x, true)); } -//TODO{lukeiwanski}: Implement BatchMatMul for SYCL +// TODO{lukeiwanski}: Implement BatchMatMul for SYCL #ifndef TENSORFLOW_USE_SYCL TEST_F(MathGradTest, BatchMatMul_00) { auto x = test::AsTensor({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, @@ -992,7 +991,7 @@ TEST_F(MathGradTest, BatchMatMul_11) { test::ExpectClose(dx, BatchMatMul(y, true, dz, true)); test::ExpectClose(dy, BatchMatMul(dz, true, x, true)); } -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TEST_F(MathGradTest, Sum_dim0) { auto x = test::AsTensor({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 61db896c51..b06cb2a241 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -235,7 +235,9 @@ value is computed as \\( \sqrt{a^2 + b^2}\\). .Attr("T: {half, float, double, complex64, complex128}") \ .SetShapeFn(shape_inference::UnchangedShape) -REGISTER_OP("Neg").UNARY().Doc(R"doc( +REGISTER_OP("Neg") + .UNARY() + .Doc(R"doc( Computes numerical negative value element-wise. I.e., \\(y = -x\\). )doc"); @@ -258,155 +260,217 @@ is the corresponding input gradient. )doc") .Deprecated(17, "Use ReciprocalGrad"); -REGISTER_OP("Reciprocal").UNARY().Doc(R"doc( +REGISTER_OP("Reciprocal") + .UNARY() + .Doc(R"doc( Computes the reciprocal of x element-wise. I.e., \\(y = 1 / x\\). )doc"); -REGISTER_OP("ReciprocalGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("ReciprocalGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the inverse of `x` wrt its input. Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Square").UNARY().Doc(R"doc( +REGISTER_OP("Square") + .UNARY() + .Doc(R"doc( Computes square of x element-wise. I.e., \\(y = x * x = x^2\\). )doc"); -REGISTER_OP("Sqrt").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sqrt") + .UNARY_COMPLEX() + .Doc(R"doc( Computes square root of x element-wise. I.e., \\(y = \sqrt{x} = x^{1/2}\\). )doc"); -REGISTER_OP("SqrtGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("SqrtGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the sqrt of `x` wrt its input. Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Rsqrt").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Rsqrt") + .UNARY_COMPLEX() + .Doc(R"doc( Computes reciprocal of square root of x element-wise. I.e., \\(y = 1 / \sqrt{x}\\). )doc"); -REGISTER_OP("Round").UNARY().Doc(R"doc( +REGISTER_OP("Round") + .UNARY() + .Doc(R"doc( Rounds the values of a tensor to the nearest integer, element-wise. Rounds half to even. Also known as bankers rounding. If you want to round according to the current system rounding mode use std::cint. )doc"); -REGISTER_OP("RsqrtGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("RsqrtGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the rsqrt of `x` wrt its input. Specifically, `grad = dy * -0.5 * y^3`, where `y = rsqrt(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Exp").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Exp") + .UNARY_COMPLEX() + .Doc(R"doc( Computes exponential of x element-wise. \\(y = e^x\\). )doc"); -REGISTER_OP("Expm1").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Expm1") + .UNARY_COMPLEX() + .Doc(R"doc( Computes exponential of x - 1 element-wise. I.e., \\(y = (\exp x) - 1\\). )doc"); -REGISTER_OP("Log").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Log") + .UNARY_COMPLEX() + .Doc(R"doc( Computes natural logarithm of x element-wise. I.e., \\(y = \log_e x\\). )doc"); -REGISTER_OP("Log1p").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Log1p") + .UNARY_COMPLEX() + .Doc(R"doc( Computes natural logarithm of (1 + x) element-wise. I.e., \\(y = \log_e (1 + x)\\). )doc"); -REGISTER_OP("Sinh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sinh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes hyperbolic sine of x element-wise. )doc"); -REGISTER_OP("Cosh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Cosh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes hyperbolic cosine of x element-wise. )doc"); -REGISTER_OP("Tanh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Tanh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes hyperbolic tangent of `x` element-wise. )doc"); -REGISTER_OP("Asinh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Asinh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes inverse hyperbolic sine of x element-wise. )doc"); -REGISTER_OP("Acosh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Acosh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes inverse hyperbolic cosine of x element-wise. )doc"); -REGISTER_OP("Atanh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Atanh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes inverse hyperbolic tangent of x element-wise. )doc"); -REGISTER_OP("TanhGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("TanhGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the tanh of `x` wrt its input. Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Lgamma").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Lgamma") + .UNARY_REAL() + .Doc(R"doc( Computes the log of the absolute value of `Gamma(x)` element-wise. )doc"); -REGISTER_OP("Digamma").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Digamma") + .UNARY_REAL() + .Doc(R"doc( Computes Psi, the derivative of Lgamma (the log of the absolute value of `Gamma(x)`), element-wise. )doc"); -REGISTER_OP("Erf").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Erf") + .UNARY_REAL() + .Doc(R"doc( Computes the Gauss error function of `x` element-wise. )doc"); -REGISTER_OP("Erfc").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Erfc") + .UNARY_REAL() + .Doc(R"doc( Computes the complementary error function of `x` element-wise. )doc"); -REGISTER_OP("Sigmoid").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sigmoid") + .UNARY_COMPLEX() + .Doc(R"doc( Computes sigmoid of `x` element-wise. Specifically, `y = 1 / (1 + exp(-x))`. )doc"); -REGISTER_OP("SigmoidGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("SigmoidGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient of the sigmoid of `x` wrt its input. Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Sin").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sin") + .UNARY_COMPLEX() + .Doc(R"doc( Computes sin of x element-wise. )doc"); -REGISTER_OP("Cos").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Cos") + .UNARY_COMPLEX() + .Doc(R"doc( Computes cos of x element-wise. )doc"); -REGISTER_OP("Tan").UNARY().Doc(R"doc( +REGISTER_OP("Tan") + .UNARY() + .Doc(R"doc( Computes tan of x element-wise. )doc"); -REGISTER_OP("Asin").UNARY().Doc(R"doc( +REGISTER_OP("Asin") + .UNARY() + .Doc(R"doc( Computes asin of x element-wise. )doc"); -REGISTER_OP("Acos").UNARY().Doc(R"doc( +REGISTER_OP("Acos") + .UNARY() + .Doc(R"doc( Computes acos of x element-wise. )doc"); -REGISTER_OP("Atan").UNARY().Doc(R"doc( +REGISTER_OP("Atan") + .UNARY() + .Doc(R"doc( Computes atan of x element-wise. )doc"); @@ -942,28 +1006,36 @@ beta function. .Attr("T: realnumbertype") \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) -REGISTER_OP("Less").COMPARISON().Doc(R"doc( +REGISTER_OP("Less") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x < y) element-wise. *NOTE*: `Less` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("LessEqual").COMPARISON().Doc(R"doc( +REGISTER_OP("LessEqual") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x <= y) element-wise. *NOTE*: `LessEqual` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("Greater").COMPARISON().Doc(R"doc( +REGISTER_OP("Greater") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x > y) element-wise. *NOTE*: `Greater` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("GreaterEqual").COMPARISON().Doc(R"doc( +REGISTER_OP("GreaterEqual") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x >= y) element-wise. *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting @@ -985,14 +1057,18 @@ Returns the truth value of (x >= y) element-wise. "quint8, qint8, qint32, string, bool, complex128}") \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) -REGISTER_OP("Equal").EQUALITY_COMPARISON().Doc(R"doc( +REGISTER_OP("Equal") + .EQUALITY_COMPARISON() + .Doc(R"doc( Returns the truth value of (x == y) element-wise. *NOTE*: `Equal` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("NotEqual").EQUALITY_COMPARISON().Doc(R"doc( +REGISTER_OP("NotEqual") + .EQUALITY_COMPARISON() + .Doc(R"doc( Returns the truth value of (x != y) element-wise. *NOTE*: `NotEqual` supports broadcasting. More about broadcasting @@ -1030,14 +1106,18 @@ Returns the truth value of NOT x element-wise. .SetIsCommutative() \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) -REGISTER_OP("LogicalAnd").BINARY_LOGICAL().Doc(R"doc( +REGISTER_OP("LogicalAnd") + .BINARY_LOGICAL() + .Doc(R"doc( Returns the truth value of x AND y element-wise. *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("LogicalOr").BINARY_LOGICAL().Doc(R"doc( +REGISTER_OP("LogicalOr") + .BINARY_LOGICAL() + .Doc(R"doc( Returns the truth value of x OR y element-wise. *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting @@ -1977,12 +2057,12 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t, T limit = limit_t->scalar()(); T delta = delta_t->scalar()(); if (start > limit && delta > 0) { - return errors::InvalidArgument( - "Requires start <= limit when delta > 0: ", start, "/", limit); + return errors::InvalidArgument("Requires start <= limit when delta > 0: ", + start, "/", limit); } if (start < limit && delta < 0) { - return errors::InvalidArgument( - "Requires start >= limit when delta < 0: ", start, "/", limit); + return errors::InvalidArgument("Requires start >= limit when delta < 0: ", + start, "/", limit); } if (delta == 0) { return errors::InvalidArgument("Requires delta != 0"); diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 1d26660a4b..de059a3e7e 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -2176,9 +2176,9 @@ Status TopKShapeFn(InferenceContext* c) { DimensionHandle last_dim = c->Dim(input, -1); if (c->ValueKnown(last_dim) && c->ValueKnown(k_dim) && c->Value(last_dim) < c->Value(k_dim)) { - return errors::InvalidArgument( - "input must have last dimension >= k = ", c->Value(k_dim), " but is ", - c->Value(last_dim)); + return errors::InvalidArgument("input must have last dimension >= k = ", + c->Value(k_dim), " but is ", + c->Value(last_dim)); } // Replace last_dim with k_dim. @@ -2278,9 +2278,9 @@ REGISTER_OP("NthElement") DimensionHandle last_dim = c->Dim(input, -1); if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) && c->Value(last_dim) <= c->Value(n_dim)) { - return errors::InvalidArgument( - "Input must have last dimension > n = ", c->Value(n_dim), " but is ", - c->Value(last_dim)); + return errors::InvalidArgument("Input must have last dimension > n = ", + c->Value(n_dim), " but is ", + c->Value(last_dim)); } // Reduce last_dim for output tensor diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc index 94ecf4d5db..1b17a7cda6 100644 --- a/tensorflow/core/ops/nn_ops_test.cc +++ b/tensorflow/core/ops/nn_ops_test.cc @@ -95,14 +95,13 @@ TEST(NNOpsTest, NthElement_ShapeFn) { INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]"); INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]"); - INFER_ERROR("Input must have last dimension > n = 20 but is 1", op, - "[1];[]"); + INFER_ERROR("Input must have last dimension > n = 20 but is 1", op, "[1];[]"); INFER_ERROR("Input must have last dimension > n = 20 but is 20", op, "[1,2,3,20];[]"); n_t = test::AsScalar(-1); INFER_ERROR( - "Dimension size, given by scalar input 1, must be non-negative but is -1", - op, "[1,2,3,4];[]"); + "Dimension size, given by scalar input 1, must be non-negative but is -1", + op, "[1,2,3,4];[]"); } TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) { @@ -386,9 +385,8 @@ TEST(NNOpsTest, Dilation2DBackpropFilter_ShapeFn) { } TEST(NNOpsTest, MergeBothInputs_ShapeFn) { - for (const char* op_name : - {"ReluGrad", "Relu6Grad", "EluGrad", "SeluGrad", "SoftplusGrad", - "SoftsignGrad"}) { + for (const char* op_name : {"ReluGrad", "Relu6Grad", "EluGrad", "SeluGrad", + "SoftplusGrad", "SoftsignGrad"}) { ShapeInferenceTestOp op(op_name); INFER_OK(op, "?;?", "in0|in1"); diff --git a/tensorflow/core/ops/sparse_ops_test.cc b/tensorflow/core/ops/sparse_ops_test.cc index ea49f1a199..0df3320484 100644 --- a/tensorflow/core/ops/sparse_ops_test.cc +++ b/tensorflow/core/ops/sparse_ops_test.cc @@ -187,8 +187,8 @@ TEST(SparseOpsTest, SparseTensorDenseMatMul_ShapeFn) { // second output dim comes from b, depending on adjoint_b value. INFER_OK(op, "?;?;?;?", "[?,?]"); - INFER_OK(op, "?;?;?;[?,?]", "[?,d3_1]"); // use d3_1, !adjoint_b. - INFER_OK(op, "?;?;?;[1,2]", "[?,d3_1]"); // use d3_1, !adjoint_b. + INFER_OK(op, "?;?;?;[?,?]", "[?,d3_1]"); // use d3_1, !adjoint_b. + INFER_OK(op, "?;?;?;[1,2]", "[?,d3_1]"); // use d3_1, !adjoint_b. INFER_OK(op, "?;?;[2];[1,2]", "[?,d3_1]"); // use d3_1, !adjoint_b. set_adjoints(false, true); diff --git a/tensorflow/core/ops/stateless_random_ops.cc b/tensorflow/core/ops/stateless_random_ops.cc index b222b5b241..7c00fdb99f 100644 --- a/tensorflow/core/ops/stateless_random_ops.cc +++ b/tensorflow/core/ops/stateless_random_ops.cc @@ -45,7 +45,8 @@ static Status StatelessShape(shape_inference::InferenceContext* context) { .SetShapeFn(StatelessShape) // This op is exposed through contrib/stateless only. The interface may change. -REGISTER_STATELESS_OP("StatelessRandomUniform").Doc(R"doc( +REGISTER_STATELESS_OP("StatelessRandomUniform") + .Doc(R"doc( Outputs deterministic pseudorandom random values from a uniform distribution. The generated values follow a uniform distribution in the range `[0, 1)`. The @@ -60,7 +61,8 @@ output: Random values with specified shape. )doc"); // This op is exposed through contrib/stateless only. The interface may change. -REGISTER_STATELESS_OP("StatelessRandomNormal").Doc(R"doc( +REGISTER_STATELESS_OP("StatelessRandomNormal") + .Doc(R"doc( Outputs deterministic pseudorandom values from a normal distribution. The generated values will have mean 0 and standard deviation 1. @@ -74,7 +76,8 @@ output: Random values with specified shape. )doc"); // This op is exposed through contrib/stateless only. The interface may change. -REGISTER_STATELESS_OP("StatelessTruncatedNormal").Doc(R"doc( +REGISTER_STATELESS_OP("StatelessTruncatedNormal") + .Doc(R"doc( Outputs deterministic pseudorandom values from a truncated normal distribution. The generated values follow a normal distribution with mean 0 and standard -- GitLab From c6aecff757f36b9446f742a750eac9cc768a230b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 25 Oct 2017 00:42:18 +0000 Subject: [PATCH 378/573] Add the option to do an incremental clang style check. Signed-off-by: Yong Tang --- tensorflow/tools/ci_build/ci_sanity.sh | 62 ++++++++++++++++++- .../tools/ci_build/clang_format_check.sh | 36 ----------- 2 files changed, 61 insertions(+), 37 deletions(-) delete mode 100755 tensorflow/tools/ci_build/clang_format_check.sh diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 7cf97dacf0..e1757c6d15 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -426,8 +426,68 @@ do_code_link_check() { tensorflow/tools/ci_build/code_link_check.sh } +# List .h|.cc files changed in the last non-merge git commit that still exist, +# i.e., not removed. +# Usage: get_clang_files_to_check [--incremental] +get_clang_files_to_check() { + if [[ "$1" == "--incremental" ]]; then + CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \ + grep '.*\.h$\|.*\.cc$') + + # Do not include files removed in the last non-merge commit. + CLANG_FILES="" + for CLANG_FILE in ${CHANGED_CLANG_FILES}; do + if [[ -f "${CLANG_FILE}" ]]; then + CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}" + fi + done + + echo "${CLANG_FILES}" + else + find tensorflow -name '*.h' -o -name '*.cc' + fi +} + do_clang_format_check() { - CLANG_FORMAT=clang-format-3.8 tensorflow/tools/ci_build/clang_format_check.sh + if [[ $# != "0" ]] && [[ $# != "1" ]]; then + echo "Invalid syntax when invoking do_clang_format_check" + echo "Usage: do_clang_format_check [--incremental]" + return 1 + fi + + if [[ "$1" == "--incremental" ]]; then + CLANG_SRC_FILES=$(get_clang_files_to_check --incremental) + + if [[ -z "${CLANG_SRC_FILES}" ]]; then + echo "do_clang_format_check will NOT run due to --incremental flag and "\ +"due to the absence of Python code changes in the last commit." + return 0 + fi + elif [[ -z "$1" ]]; then + CLANG_SRC_FILES=$(get_clang_files_to_check) + else + echo "Invalid syntax for invoking do_clang_format_check" + echo "Usage: do_clang_format_check [--incremental]" + return 1 + fi + + CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8} + + success=1 + for filename in $CLANG_SRC_FILES; do + $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null + if [ ! $? -eq 0 ]; then + success=0 + echo File $filename is not properly formatted with "clang-format "\ +"--style=google" + fi + done + + if [ $success == 0 ]; then + echo Clang format check fails. + exit 1 + fi + echo Clang format check success. } do_check_load_py_test() { diff --git a/tensorflow/tools/ci_build/clang_format_check.sh b/tensorflow/tools/ci_build/clang_format_check.sh deleted file mode 100755 index 5f252d7f35..0000000000 --- a/tensorflow/tools/ci_build/clang_format_check.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# please run this at root directory of tensorflow -success=1 - -CLANG_FORMAT=${CLANG_FORMAT:-clang-format} - -# only tensorflow/core/ops is checked at the moment for experimental purpose -for filename in $(find tensorflow/core/ops -name *.h -o -name *.cc); do - $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null - if [ ! $? -eq 0 ]; then - success=0 - echo File $filename is not properly formatted with "clang-format --style=google" - fi -done - -if [ $success == 0 ]; then - echo Clang format check fails. - exit 1 -fi - -echo Clang format check success. -- GitLab From 8087e67252bca4075e59ab75023826dae23dfb74 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Wed, 25 Oct 2017 15:50:02 -0700 Subject: [PATCH 379/573] [XLA] Remove dead kUpdate opcode. PiperOrigin-RevId: 173462881 --- tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 1 - tensorflow/compiler/xla/service/hlo_instruction.cc | 3 --- tensorflow/compiler/xla/service/hlo_matchers.h | 1 - tensorflow/compiler/xla/service/hlo_opcode.cc | 3 --- tensorflow/compiler/xla/service/hlo_opcode.h | 1 - tensorflow/compiler/xla/service/inliner.cc | 3 +-- tensorflow/compiler/xla/service/instruction_fusion.cc | 1 - tensorflow/compiler/xla/service/shape_inference.cc | 10 ---------- tensorflow/compiler/xla/service/user_computation.cc | 2 -- tensorflow/compiler/xla/tools/parser/hlo_parser.cc | 1 - tensorflow/compiler/xla/xla_data.proto | 4 ---- 11 files changed, 1 insertion(+), 29 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index 24e390529e..ed94a5be91 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -814,7 +814,6 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kPad: case HloOpcode::kReshape: case HloOpcode::kReverse: - case HloOpcode::kUpdate: return kGreen; case HloOpcode::kConvolution: case HloOpcode::kDot: diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 8e52d131a6..d53ac221d1 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -1156,7 +1156,6 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( break; case HloOpcode::kRecv: case HloOpcode::kSend: - case HloOpcode::kUpdate: case HloOpcode::kIndex: case HloOpcode::kTrace: LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_); @@ -1541,7 +1540,6 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kInfeed: case HloOpcode::kOutfeed: case HloOpcode::kSort: - case HloOpcode::kUpdate: case HloOpcode::kSend: case HloOpcode::kRecv: return false; @@ -2265,7 +2263,6 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { // These opcodes are not handled here. case HloOpcode::kIndex: case HloOpcode::kTrace: - case HloOpcode::kUpdate: break; } return Unimplemented("unhandled HloOpcode for DfsHloVisitor: %s", diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h index d1ae5f776d..5440ed2eda 100644 --- a/tensorflow/compiler/xla/service/hlo_matchers.h +++ b/tensorflow/compiler/xla/service/hlo_matchers.h @@ -115,7 +115,6 @@ HLO_MATCHER(Tanh); HLO_MATCHER(Trace); HLO_MATCHER(Transpose); HLO_MATCHER(Tuple); -HLO_MATCHER(Update); HLO_MATCHER(While); #undef HLO_MATCHER } // namespace opcode_matchers diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc index db3abeab22..e9000a8462 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.cc +++ b/tensorflow/compiler/xla/service/hlo_opcode.cc @@ -173,8 +173,6 @@ string HloOpcodeString(HloOpcode opcode) { return "transpose"; case HloOpcode::kTuple: return "tuple"; - case HloOpcode::kUpdate: - return "update"; case HloOpcode::kWhile: return "while"; } @@ -254,7 +252,6 @@ StatusOr StringToHloOpcode(const string& opcode_name) { {"trace", HloOpcode::kTrace}, {"transpose", HloOpcode::kTranspose}, {"tuple", HloOpcode::kTuple}, - {"update", HloOpcode::kUpdate}, {"while", HloOpcode::kWhile}}); auto it = opcode_map->find(opcode_name); if (it == opcode_map->end()) { diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h index 4593df671e..c603c57e62 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.h +++ b/tensorflow/compiler/xla/service/hlo_opcode.h @@ -101,7 +101,6 @@ enum class HloOpcode { kTrace, kTranspose, kTuple, - kUpdate, kWhile, }; diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc index 6ea0f127d5..40df0dc355 100644 --- a/tensorflow/compiler/xla/service/inliner.cc +++ b/tensorflow/compiler/xla/service/inliner.cc @@ -76,8 +76,7 @@ Status InlinerVisitor::HandleMap( // Only inlining functions that are simply a single operation until a better // profitability model for inlining is defined. if (hlo_query::AllOperandsAreParameters(root)) { - if (root.opcode() == HloOpcode::kUpdate || - root.opcode() == HloOpcode::kFusion || + if (root.opcode() == HloOpcode::kFusion || root.opcode() == HloOpcode::kIndex || root.opcode() == HloOpcode::kParameter || root.opcode() == HloOpcode::kTrace) { diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc index 7e46d79ba4..0271f41697 100644 --- a/tensorflow/compiler/xla/service/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/instruction_fusion.cc @@ -105,7 +105,6 @@ namespace xla { case HloOpcode::kSort: case HloOpcode::kTanh: case HloOpcode::kTrace: - case HloOpcode::kUpdate: case HloOpcode::kWhile: case HloOpcode::kSend: case HloOpcode::kRecv: diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index 1df1022442..e41b7607c5 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -136,8 +136,6 @@ TernaryOperation OpcodeToTernaryOperation(HloOpcode opcode) { return TRIOP_CLAMP; case HloOpcode::kSelect: return TRIOP_SELECT; - case HloOpcode::kUpdate: - return TRIOP_UPDATE; default: LOG(FATAL) << "unhandled opcode " << opcode; } @@ -822,14 +820,6 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( return InferClampShape(lhs, rhs, ehs); case TRIOP_SELECT: return InferSelectShape(lhs, rhs, ehs); - case TRIOP_UPDATE: - TF_RETURN_IF_ERROR( - ExpectNotTupleOrOpaque(lhs, "lhs of ternary operation")); - TF_RETURN_IF_ERROR( - ExpectNotTupleOrOpaque(rhs, "rhs of ternary operation")); - TF_RETURN_IF_ERROR( - ExpectNotTupleOrOpaque(ehs, "ehs of ternary operation")); - return lhs; default: return InvalidArgument("unknown operation %s", TernaryOperation_Name(operation).c_str()); diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index 065d2580c6..d818830f98 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -133,8 +133,6 @@ HloOpcode TernaryOperationToHloOpcode(TernaryOperation triop) { return HloOpcode::kClamp; case TRIOP_SELECT: return HloOpcode::kSelect; - case TRIOP_UPDATE: - return HloOpcode::kUpdate; default: LOG(FATAL) << "unhandled operation " << triop; } diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc index a075d9057f..f4af03cc2f 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -400,7 +400,6 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, case HloOpcode::kInfeed: case HloOpcode::kOutfeed: case HloOpcode::kBatchNormGrad: - case HloOpcode::kUpdate: case HloOpcode::kIndex: case HloOpcode::kTrace: return TokenError(StrCat("parsing not yet implemented for op: ", diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 7ad61fab81..0efa3d0014 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -759,10 +759,6 @@ enum TernaryOperation { // true and operand1 if the predicate is false. TRIOP_SELECT = 1; - // Updates operand0 at index operand1 with value operand2 and outputs the - // updated value. - TRIOP_UPDATE = 2; - // Given a min, max and an operand returns the operand if between min and max, // else returns min if operand is less than min or max if operand is greater // than max. -- GitLab From 5cd1ba5560be0af52cad5184c578be17ed7e8188 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 16:00:28 -0700 Subject: [PATCH 380/573] Update documentation for SVD to make the differences with numpy.linalg.svd clearer. PiperOrigin-RevId: 173464390 --- tensorflow/python/ops/linalg_ops.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index 1d917c22cc..2cb467c891 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -368,11 +368,11 @@ def self_adjoint_eigvals(tensor, name=None): def svd(tensor, full_matrices=False, compute_uv=True, name=None): - """Computes the singular value decompositions of one or more matrices. + r"""Computes the singular value decompositions of one or more matrices. Computes the SVD of each inner matrix in `tensor` such that - `tensor[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, - :])` + `tensor[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * + transpose(conj(v[..., :, :]))` ```python # a is a tensor. @@ -406,9 +406,25 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None): `[..., N, N]`. Not returned if `compute_uv` is `False`. @compatibility(numpy) - Mostly equivalent to numpy.linalg.svd, except that the order of output - arguments here is `s`, `u`, `v` when `compute_uv` is `True`, as opposed to - `u`, `s`, `v` for numpy.linalg.svd. + Mostly equivalent to numpy.linalg.svd, except that + * The order of output arguments here is `s`, `u`, `v` when `compute_uv` is + `True`, as opposed to `u`, `s`, `v` for numpy.linalg.svd. + * full_matrices is `False` by default as opposed to `True` for + numpy.linalg.svd. + * tf.linalg.svd uses the standard definition of the SVD + \\(A = U \Sigma V^H\\), such that the left singular vectors of `a` are + the columns of `u`, while the right singular vectors of `a` are the + columns of `v`. On the other hand, numpy.linalg.svd returns the adjoint + \\(V^H\\) as the third output argument. + ```python + import tensorflow as tf + import numpy as np + s, u, v = tf.linalg.svd(a) + tf_a_approx = tf.matmul(u, tf.matmul(tf.linalg.diag(s), v, adjoint_v=True)) + u, s, v_adj = np.linalg.svd(a, full_matrices=False) + np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj)) + # tf_a_approx and np_a_approx should be numerically close. + ```` @end_compatibility """ # pylint: disable=protected-access -- GitLab From 7efbfc22850394c94785885d3b06cdd71124e5d6 Mon Sep 17 00:00:00 2001 From: Alina Sbirlea Date: Wed, 25 Oct 2017 16:01:35 -0700 Subject: [PATCH 381/573] Fix test: Call instruction was added to builder instead of call_builder. Triggers the assertion in MakeInstructionPostOrder when dumping out the module. PiperOrigin-RevId: 173464579 --- tensorflow/compiler/xla/service/algebraic_simplifier_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index 57be144b36..3df50080d1 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -2127,7 +2127,7 @@ TEST_F(AlgebraicSimplifierTest, IteratorInvalidation) { HloInstruction::CreateConstant(Literal::CreateR1({0.0f}))); HloInstruction* one = call_builder.AddInstruction( HloInstruction::CreateConstant(Literal::CreateR1({1.0f}))); - builder.AddInstruction( + call_builder.AddInstruction( HloInstruction::CreateCall(r1f32, {zero, one}, dot_computation.get())); auto module = CreateNewModule(); -- GitLab From 641e0df038f14be471ff54df626081515d25a741 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 16:22:58 -0700 Subject: [PATCH 382/573] Should not affect public. PiperOrigin-RevId: 173467560 --- tensorflow/python/training/monitored_session.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py index 2dd2114af0..dea62d27ba 100644 --- a/tensorflow/python/training/monitored_session.py +++ b/tensorflow/python/training/monitored_session.py @@ -349,8 +349,10 @@ def MonitoredTrainingSession(master='', # pylint: disable=invalid-name config=config) if checkpoint_dir: - all_hooks.append(basic_session_run_hooks.StepCounterHook( - output_dir=checkpoint_dir, every_n_steps=log_step_count_steps)) + if log_step_count_steps and log_step_count_steps > 0: + all_hooks.append( + basic_session_run_hooks.StepCounterHook( + output_dir=checkpoint_dir, every_n_steps=log_step_count_steps)) if (save_summaries_steps and save_summaries_steps > 0) or ( save_summaries_secs and save_summaries_secs > 0): -- GitLab From 9a0d94136b9a1d0ff62a81b50fb8c3dbb0c3b4a2 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 25 Oct 2017 23:30:45 +0000 Subject: [PATCH 383/573] Always enable --incremental. Signed-off-by: Yong Tang --- tensorflow/tools/ci_build/ci_sanity.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index e1757c6d15..1e1fd7db6b 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -464,7 +464,9 @@ do_clang_format_check() { return 0 fi elif [[ -z "$1" ]]; then - CLANG_SRC_FILES=$(get_clang_files_to_check) + # TODO (yongtang): Always pass --incremental until all files have + # been sanitized gradually. Then this --incremental could be removed. + CLANG_SRC_FILES=$(get_clang_files_to_check --incremental) else echo "Invalid syntax for invoking do_clang_format_check" echo "Usage: do_clang_format_check [--incremental]" -- GitLab From 1f696bcf4b159624411dbb1b9f4a206aa0cd1c43 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 25 Oct 2017 16:34:26 -0700 Subject: [PATCH 384/573] Explicitly disables variable inplace assignment methods. Need a story for eager, graph functions (both easy) and graphs (hard). PiperOrigin-RevId: 173468930 --- .../python/ops/resource_variable_ops.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 386fd204b6..439fa84238 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -768,6 +768,27 @@ class ResourceVariable(variables.Variable): else: return self.value() + def __iadd__(self, unused_other): + raise RuntimeError("Variable += value not supported.") + + def __isub__(self, unused_other): + raise RuntimeError("Variable -= value not supported.") + + def __imul__(self, unused_other): + raise RuntimeError("Variable *= value not supported.") + + def __idiv__(self, unused_other): + raise RuntimeError("Variable /= value not supported.") + + def __itruediv__(self, unused_other): + raise RuntimeError("Variable /= value not supported.") + + def __irealdiv__(self, unused_other): + raise RuntimeError("Variable /= value not supported.") + + def __ipow__(self, unused_other): + raise RuntimeError("Variable **= value not supported.") + def _dense_var_to_tensor(var, dtype=None, name=None, as_ref=False): return var._dense_var_to_tensor(dtype=dtype, name=name, as_ref=as_ref) # pylint: disable=protected-access -- GitLab From b4e09b48617ed27f528ee2b253c01a6e4698c2e3 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 25 Oct 2017 16:48:26 -0700 Subject: [PATCH 385/573] Internal change. PiperOrigin-RevId: 173471032 --- tensorflow/java/BUILD | 2 +- .../opensource_only/arm_neon_2_x86_sse.BUILD | 16 ++++++++++++++++ tensorflow/workspace.bzl | 10 ++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index d74cb32c5a..c0563da06d 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -289,7 +289,7 @@ tf_java_test( # test_class = "org.tensorflow.processor.OperatorProcessorTest", # deps = [ # ":processor_library", -# "@junit", +# "//third_party/java/junit", # "@com_google_testing_compile", # "@com_google_truth", # ], diff --git a/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD b/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD new file mode 100644 index 0000000000..6c641a7f4e --- /dev/null +++ b/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD @@ -0,0 +1,16 @@ +# Description: +# NEON2SSE - a header file redefining ARM Neon intrinsics in terms of SSE intrinsics +# allowing neon code to compile and run on x64/x86 workstantions. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # 3-Clause BSD + +exports_files([ + "LICENSE", +]) + +cc_library( + name = "arm_neon_2_x86_sse", + hdrs = ["NEON_2_SSE.h"], +) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index a14469a0be..f5006ad55d 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -770,3 +770,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "d58bb2d6c8603f600d522b6104d6192a65339aa26cbba9f11ff5c4b36dedb928", strip_prefix = "bazel-toolchains-af4681c3d19f063f090222ec3d04108c4e0ca255", ) + + native.new_http_archive( + name = "arm_neon_2_x86_sse", + sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5", + strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d", + urls = [ + "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", + ], + build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")), + ) -- GitLab From a80b9297f330be6777a23e2e3a3b6e21097d1926 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 16:56:06 -0700 Subject: [PATCH 386/573] Add device assignment export to graph dumpers. PiperOrigin-RevId: 173472156 --- tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 7 ++++++- tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index ed94a5be91..20ec7dfe2f 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -926,6 +926,9 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { [](int64 stride) { return stride == 1; }) ? "" : StrCat("stride=", VectorString(instr->slice_strides())); + case HloOpcode::kSend: + case HloOpcode::kRecv: + return StrCat("channel_id=", instr->channel_id()); default: return ""; } @@ -935,7 +938,9 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { if (!opcode_specific_info.empty()) { lines.push_back(opcode_specific_info); } - + if (instr->device_assignment().has_device()) { + lines.push_back(StrCat("device=", instr->device_assignment().device())); + } // Show the shape and layout of the instruction, unless it's an inlined fusion // node -- there the shape and layout is present in the output node. if (instr->opcode() != HloOpcode::kFusion || diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc index 3f6d89f24f..2007a8f11d 100644 --- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc +++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc @@ -56,6 +56,8 @@ TensorShapeProto GetTensorShape(const HloInstruction* instruction) { return tensor_shape; } +string GetDeviceName(int device) { return StrCat("/device/XLA:", device); } + } // namespace void CleanNodeName(string* name) { @@ -178,6 +180,10 @@ void HloTfGraphBuilder::SetNodeAttrs(const HloInstruction* instruction, case HloOpcode::kCustomCall: attrs["custom_call_target"].set_s(instruction->custom_call_target()); break; + case HloOpcode::kSend: + case HloOpcode::kRecv: + attrs["channel_id"].set_i(instruction->channel_id()); + break; default: break; } @@ -192,6 +198,10 @@ Status HloTfGraphBuilder::AddInstruction(const HloInstruction* instruction) { NodeDef* node_def = graph_def_.add_node(); node_def->set_name(GetNodeNameForInstruction(instruction)); node_def->set_op(GetOpDefName(instruction)); + if (instruction->device_assignment().has_device()) { + node_def->set_device( + GetDeviceName(instruction->device_assignment().device())); + } SetNodeAttrs(instruction, node_def); if (instruction->opcode() == HloOpcode::kFusion) { for (auto* fused_instruction : instruction->fused_instructions()) { -- GitLab From 3315397770b5304f529e686c635a9436f32aa61d Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 25 Oct 2017 16:59:48 -0700 Subject: [PATCH 387/573] Support general callables in backprop. PiperOrigin-RevId: 173472596 --- tensorflow/python/eager/backprop.py | 13 ++++++++++++- tensorflow/python/eager/backprop_test.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index bdc4ce3252..6ede02dbcd 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -400,7 +400,18 @@ def implicit_grad(f): def _get_arg_spec(f, params, param_args): - args = tf_inspect.getargspec(f).args + """The positions of the parameters of f to be differentiated in param_args.""" + try: + args = tf_inspect.getargspec(f).args + except TypeError as e: + # TypeError can happen when f is a callable object. + if params is None: + return range(len(param_args)) + elif all(isinstance(x, int) for x in params): + return params + raise ValueError("Either callable provided is not a function or could not " + "inspect its arguments by name: %s. Original error: %s" + % (f, e)) if params is None: if not args: return range(len(param_args)) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 628f254b18..d18df4dffb 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -16,6 +16,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + import numpy as np from tensorflow.python import pywrap_tensorflow @@ -389,6 +391,16 @@ class BackpropTest(test.TestCase): grad = backprop.gradients_function(f) self.assertAllEqual(grad(1.0)[0], 2.0) + def testPartial(self): + + def f(x, y): + return x * y + + part = functools.partial(f, constant_op.constant(2.0)) + self.assertAllEqual( + backprop.gradients_function(part)(constant_op.constant(1.0))[0], + 2.0) + def testExceptionSafety(self): def f(unused_x): -- GitLab From 176a743ba351ddadee3f1f168a6eb32315264e65 Mon Sep 17 00:00:00 2001 From: Sergio Guadarrama Date: Wed, 25 Oct 2017 17:02:37 -0700 Subject: [PATCH 388/573] Expose name and func properties of Template. PiperOrigin-RevId: 173473015 --- tensorflow/python/ops/template.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py index fab808a167..24ef70c6f4 100644 --- a/tensorflow/python/ops/template.py +++ b/tensorflow/python/ops/template.py @@ -279,6 +279,16 @@ class Template(object): self._variables_created = True return result + @property + def name(self): + """Returns the name given to this Template.""" + return self._name + + @property + def func(self): + """Returns the func given to this Template.""" + return self._func + @property def variable_scope(self): """Returns the variable scope object created by this Template.""" -- GitLab From ee501c6b98cfea07897637b403d40d0a28c4bc6e Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 25 Oct 2017 17:08:29 -0700 Subject: [PATCH 389/573] Proper destructuring of function arguments. PiperOrigin-RevId: 173473904 --- tensorflow/python/eager/function.py | 29 ++++++++---------------- tensorflow/python/eager/function_test.py | 8 +++++++ 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 5afc9d295e..b1b1de0c41 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -328,7 +328,7 @@ class _GraphModeFunction(object): (args + self._extra_inputs), backward_function) - return self._build_call_outputs(self._returns, real_outputs) + return self._build_call_outputs(real_outputs) def __call__(self, *args): """Executes the passed function in eager mode.""" @@ -371,34 +371,25 @@ class _GraphModeFunction(object): attrs=None, ctx=ctx) - return self._build_call_outputs(self._returns, result) + return self._build_call_outputs(result) - def _build_call_outputs(self, func_outputs, result): + def _build_call_outputs(self, result): """Maps the fdef output list to actual output structure. Args: - func_outputs: The outputs originally defined by the graph function. It - could potentially be a nested structure. result: Output lists defined by FunctionDef. Returns: The actual call output. """ if self._func_outputs is None: return None - if isinstance(self._func_outputs, ops.Tensor): - return result[0] - - outputs = [] - for o in func_outputs: - vo = o - if isinstance(vo, ops.Tensor): - outputs.append(result[self._returns_to_fedf_outputs[id(vo)]]) - elif type(vo) in (tuple, list): - outputs.append(self._build_call_outputs(o, result)) - else: - outputs.append(o) - - return tuple(outputs) if type(func_outputs) is tuple else outputs + outputs_list = nest.flatten(self._func_outputs) + j = 0 + for i, o in enumerate(outputs_list): + if o is not None: + outputs_list[i] = result[j] + j += 1 + return nest.pack_sequence_as(self._func_outputs, outputs_list) def _get_defun_inputs(args): diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index b4b704401a..243efccac4 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -99,6 +99,14 @@ class FunctionTest(test.TestCase): self.assertAllEqual(g(constant_op.constant(2.0)).eval(), 5.0) + def testDict(self): + + @function.defun + def f(x): + return {'name': x + 1} + + self.assertAllEqual(f(constant_op.constant(1.0))['name'], 2.0) + def testTensorConversionWithDefun(self): @function.defun -- GitLab From 35ca57d39b9e368ef43302421db774e4ac3e3625 Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Wed, 25 Oct 2017 17:15:55 -0700 Subject: [PATCH 390/573] Bugfix: Improve numerical stability of `tf.contrib.distributions.NegativeBinomial.log_prob`. PiperOrigin-RevId: 173474795 --- .../kernel_tests/negative_binomial_test.py | 22 +++++++++++++++++++ .../python/ops/negative_binomial.py | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py index c1a74c6483..37edaa42cd 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py @@ -241,6 +241,28 @@ class NegativeBinomialTest(test.TestCase): atol=0., rtol=.02) + def testLogProbOverflow(self): + with self.test_session() as sess: + logits = np.float32([20., 30., 40.]) + total_count = np.float32(1.) + x = np.float32(0.) + nb = negative_binomial.NegativeBinomial( + total_count=total_count, logits=logits) + log_prob_ = sess.run(nb.log_prob(x)) + self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool), + np.isfinite(log_prob_)) + + def testLogProbUnderflow(self): + with self.test_session() as sess: + logits = np.float32([-90, -100, -110]) + total_count = np.float32(1.) + x = np.float32(0.) + nb = negative_binomial.NegativeBinomial( + total_count=total_count, logits=logits) + log_prob_ = sess.run(nb.log_prob(x)) + self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool), + np.isfinite(log_prob_)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/ops/negative_binomial.py b/tensorflow/contrib/distributions/python/ops/negative_binomial.py index c8c396f6f8..3a58df80da 100644 --- a/tensorflow/contrib/distributions/python/ops/negative_binomial.py +++ b/tensorflow/contrib/distributions/python/ops/negative_binomial.py @@ -167,8 +167,8 @@ class NegativeBinomial(distribution.Distribution): def _log_unnormalized_prob(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) - return (self.total_count * math_ops.log1p(-self.probs) - + x * math_ops.log(self.probs)) + return (self.total_count * math_ops.log_sigmoid(-self.logits) + + x * math_ops.log_sigmoid(self.logits)) def _log_normalization(self, x): if self.validate_args: -- GitLab From eea18bd6e5e5c8d0f0c90d6cb5b06433090c5d90 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Wed, 25 Oct 2017 17:25:27 -0700 Subject: [PATCH 391/573] Fix wall_time in Summaries 2.o It was previously storing microseconds in tf.Event.wall_time, which is a double meant for UNIX timestamps. PiperOrigin-RevId: 173475843 --- tensorflow/core/kernels/summary_interface.cc | 29 +++-- tensorflow/core/kernels/summary_interface.h | 3 +- .../core/kernels/summary_interface_test.cc | 122 +++++++++++------- 3 files changed, 93 insertions(+), 61 deletions(-) diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc index e95a4c7b89..a0b9038787 100644 --- a/tensorflow/core/kernels/summary_interface.cc +++ b/tensorflow/core/kernels/summary_interface.cc @@ -30,7 +30,7 @@ limitations under the License. namespace tensorflow { namespace { template -Status TensorValueAt(Tensor t, int index, T* out) { +Status TensorValueAt(Tensor t, int64 index, T* out) { switch (t.dtype()) { case DT_FLOAT: *out = t.flat()(index); @@ -210,20 +210,20 @@ Status NormalizeAndAddImages(const Tensor& tensor, int max_images, int h, int w, class SummaryWriterImpl : public SummaryWriterInterface { public: - SummaryWriterImpl(int max_queue, int flush_millis) + SummaryWriterImpl(int max_queue, int flush_millis, Env* env) : SummaryWriterInterface(), is_initialized_(false), max_queue_(max_queue), - flush_millis_(flush_millis) {} + flush_millis_(flush_millis), + env_(env) {} - Status Initialize(const string& logdir, const string& filename_suffix, - Env* env) { - const Status is_dir = env->IsDirectory(logdir); + Status Initialize(const string& logdir, const string& filename_suffix) { + const Status is_dir = env_->IsDirectory(logdir); if (!is_dir.ok()) { if (is_dir.code() != tensorflow::error::NOT_FOUND) { return is_dir; } - TF_RETURN_IF_ERROR(env->CreateDir(logdir)); + TF_RETURN_IF_ERROR(env_->CreateDir(logdir)); } mutex_lock ml(mu_); events_writer_ = @@ -231,7 +231,7 @@ class SummaryWriterImpl : public SummaryWriterInterface { if (!events_writer_->InitWithSuffix(filename_suffix)) { return errors::Unknown("Could not initialize events writer."); } - last_flush_ = Env::Default()->NowMicros(); + last_flush_ = env_->NowMicros(); is_initialized_ = true; return Status::OK(); } @@ -384,9 +384,9 @@ class SummaryWriterImpl : public SummaryWriterInterface { private: Status Enqueue(int64 global_step, const Summary& summary) { mutex_lock ml(mu_); - queue_.emplace_back(global_step, summary, Env::Default()->NowMicros()); + queue_.emplace_back(global_step, summary, env_->NowMicros()); if (queue_.size() >= max_queue_ || - Env::Default()->NowMicros() - last_flush_ > 1000 * flush_millis_) { + env_->NowMicros() - last_flush_ > 1000 * flush_millis_) { return InternalFlush(); } return Status::OK(); @@ -397,14 +397,14 @@ class SummaryWriterImpl : public SummaryWriterInterface { Event event; event.set_step(std::get<0>(e)); *event.mutable_summary() = std::get<1>(e); - event.set_wall_time(std::get<2>(e)); + event.set_wall_time(static_cast(std::get<2>(e)) / 1.0e6); events_writer_->WriteEvent(event); } queue_.clear(); if (!events_writer_->Flush()) { return errors::InvalidArgument("Could not flush events file."); } - last_flush_ = Env::Default()->NowMicros(); + last_flush_ = env_->NowMicros(); return Status::OK(); } @@ -412,6 +412,7 @@ class SummaryWriterImpl : public SummaryWriterInterface { const int max_queue_; const int flush_millis_; uint64 last_flush_; + Env* env_; using EventInfo = std::tuple; mutex mu_; std::vector queue_ GUARDED_BY(mu_); @@ -424,8 +425,8 @@ class SummaryWriterImpl : public SummaryWriterInterface { Status CreateSummaryWriter(int max_queue, int flush_millis, const string& logdir, const string& filename_suffix, Env* env, SummaryWriterInterface** result) { - SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis); - const Status s = w->Initialize(logdir, filename_suffix, env); + SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis, env); + const Status s = w->Initialize(logdir, filename_suffix); if (!s.ok()) { w->Unref(); *result = nullptr; diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h index ae2fbb70fe..1b5d0b2748 100644 --- a/tensorflow/core/kernels/summary_interface.h +++ b/tensorflow/core/kernels/summary_interface.h @@ -49,7 +49,8 @@ class SummaryWriterInterface : public ResourceBase { // enqueue up to max_queue summaries, and flush at least every flush_millis // milliseconds. The summaries will be written to the directory specified by // logdir and with the filename suffixed by filename_suffix. The caller owns a -// reference to result if the returned status is ok. +// reference to result if the returned status is ok. The Env object must not +// be destroyed until after the returned writer. Status CreateSummaryWriter(int max_queue, int flush_millis, const string& logdir, const string& filename_suffix, Env* env, SummaryWriterInterface** result); diff --git a/tensorflow/core/kernels/summary_interface_test.cc b/tensorflow/core/kernels/summary_interface_test.cc index 0e24e8122a..379e045ca3 100644 --- a/tensorflow/core/kernels/summary_interface_test.cc +++ b/tensorflow/core/kernels/summary_interface_test.cc @@ -28,52 +28,68 @@ limitations under the License. namespace tensorflow { namespace { -Status SummaryTestHelper( - const string& test_name, - std::function writer_fn, - std::function test_fn) { - static std::set* tests = new std::set(); - CHECK(tests->insert(test_name).second) << ": " << test_name; - - SummaryWriterInterface* writer; - Env* env = Env::Default(); - TF_CHECK_OK( - CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, env, &writer)); - core::ScopedUnref deleter(writer); - - TF_CHECK_OK(writer_fn(writer)); - TF_CHECK_OK(writer->Flush()); - - std::vector files; - TF_CHECK_OK(env->GetChildren(testing::TmpDir(), &files)); - bool found = false; - for (const string& f : files) { - if (StringPiece(f).contains(test_name)) { - if (found) { - return errors::Unknown("Found more than one file for ", test_name); +class FakeClockEnv : public EnvWrapper { + public: + FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {} + void AdvanceByMillis(const uint64 millis) { current_millis_ += millis; } + uint64 NowMicros() override { return current_millis_ * 1000; } + uint64 NowSeconds() override { return current_millis_ * 1000; } + + private: + uint64 current_millis_; +}; + +class SummaryInterfaceTest : public ::testing::Test { + protected: + Status SummaryTestHelper( + const string& test_name, + std::function writer_fn, + std::function test_fn) { + static std::set* tests = new std::set(); + CHECK(tests->insert(test_name).second) << ": " << test_name; + + SummaryWriterInterface* writer; + TF_CHECK_OK(CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, &env_, + &writer)); + core::ScopedUnref deleter(writer); + + TF_CHECK_OK(writer_fn(writer)); + TF_CHECK_OK(writer->Flush()); + + std::vector files; + TF_CHECK_OK(env_.GetChildren(testing::TmpDir(), &files)); + bool found = false; + for (const string& f : files) { + if (StringPiece(f).contains(test_name)) { + if (found) { + return errors::Unknown("Found more than one file for ", test_name); + } + found = true; + std::unique_ptr read_file; + TF_CHECK_OK(env_.NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f), + &read_file)); + io::RecordReader reader(read_file.get(), io::RecordReaderOptions()); + string record; + uint64 offset = 0; + TF_CHECK_OK( + reader.ReadRecord(&offset, + &record)); // The first event is irrelevant + TF_CHECK_OK(reader.ReadRecord(&offset, &record)); + Event e; + e.ParseFromString(record); + test_fn(e); } - found = true; - std::unique_ptr read_file; - TF_CHECK_OK(env->NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f), - &read_file)); - io::RecordReader reader(read_file.get(), io::RecordReaderOptions()); - string record; - uint64 offset = 0; - TF_CHECK_OK(reader.ReadRecord(&offset, - &record)); // The first event is irrelevant - TF_CHECK_OK(reader.ReadRecord(&offset, &record)); - Event e; - e.ParseFromString(record); - test_fn(e); } + if (!found) { + return errors::Unknown("Found no file for ", test_name); + } + return Status::OK(); } - if (!found) { - return errors::Unknown("Found no file for ", test_name); - } - return Status::OK(); -} -TEST(SummaryInterfaceTest, WriteTensor) { + FakeClockEnv env_; +}; + +TEST_F(SummaryInterfaceTest, WriteTensor) { TF_CHECK_OK(SummaryTestHelper("tensor_test", [](SummaryWriterInterface* writer) { Tensor one(DT_FLOAT, TensorShape({})); @@ -91,7 +107,7 @@ TEST(SummaryInterfaceTest, WriteTensor) { })); } -TEST(SummaryInterfaceTest, WriteScalar) { +TEST_F(SummaryInterfaceTest, WriteScalar) { TF_CHECK_OK(SummaryTestHelper( "scalar_test", [](SummaryWriterInterface* writer) { @@ -109,7 +125,7 @@ TEST(SummaryInterfaceTest, WriteScalar) { })); } -TEST(SummaryInterfaceTest, WriteHistogram) { +TEST_F(SummaryInterfaceTest, WriteHistogram) { TF_CHECK_OK(SummaryTestHelper("hist_test", [](SummaryWriterInterface* writer) { Tensor one(DT_FLOAT, TensorShape({})); @@ -127,7 +143,7 @@ TEST(SummaryInterfaceTest, WriteHistogram) { })); } -TEST(SummaryInterfaceTest, WriteImage) { +TEST_F(SummaryInterfaceTest, WriteImage) { TF_CHECK_OK(SummaryTestHelper( "image_test", [](SummaryWriterInterface* writer) { @@ -148,7 +164,7 @@ TEST(SummaryInterfaceTest, WriteImage) { })); } -TEST(SummaryInterfaceTest, WriteAudio) { +TEST_F(SummaryInterfaceTest, WriteAudio) { TF_CHECK_OK(SummaryTestHelper( "audio_test", [](SummaryWriterInterface* writer) { @@ -166,5 +182,19 @@ TEST(SummaryInterfaceTest, WriteAudio) { })); } +TEST_F(SummaryInterfaceTest, WallTime) { + env_.AdvanceByMillis(7023); + TF_CHECK_OK(SummaryTestHelper( + "wall_time_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_FLOAT, TensorShape({})); + one.scalar()() = 1.0; + TF_RETURN_IF_ERROR(writer->WriteScalar(2, one, "name")); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { EXPECT_EQ(e.wall_time(), 7.023); })); +} + } // namespace } // namespace tensorflow -- GitLab From 7924f8bce946b367a3cdcacd91ee2a6a8b29e14e Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Wed, 25 Oct 2017 17:32:57 -0700 Subject: [PATCH 392/573] Bugfix: Fix gradient of KL divergence of `tf.distributions.MultivariateNormal*`. Previous it was `nan` when the two distributions are identical. PiperOrigin-RevId: 173476896 --- .../python/kernel_tests/mvn_diag_test.py | 12 ++++++++++++ .../distributions/python/ops/mvn_linear_operator.py | 6 ++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py index 43e302475b..933756aa8e 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py @@ -289,6 +289,18 @@ class MultivariateNormalDiagTest(test.TestCase): self.assertListEqual(mvn.batch_shape.as_list(), [2, 3]) self.assertListEqual(mvn.event_shape.as_list(), [None]) + def testKLDivIdenticalGradientDefined(self): + dims = 3 + with self.test_session() as sess: + loc = array_ops.zeros([dims], dtype=dtypes.float32) + mvn = ds.MultivariateNormalDiag( + loc=loc, + scale_diag=np.ones([dims], dtype=np.float32)) + g = gradients_impl.gradients(ds.kl_divergence(mvn, mvn), loc) + g_ = sess.run(g) + self.assertAllEqual(np.ones_like(g_, dtype=np.bool), + np.isfinite(g_)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py index 251c2dbdfa..300bdd5f60 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py @@ -22,7 +22,6 @@ from tensorflow.contrib.distributions.python.ops import distribution_util from tensorflow.contrib.distributions.python.ops.bijectors import AffineLinearOperator from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.distributions import kullback_leibler from tensorflow.python.ops.distributions import normal @@ -299,7 +298,10 @@ def _kl_brute_force(a, b, name=None): def squared_frobenius_norm(x): """Helper to make KL calculation slightly more readable.""" # http://mathworld.wolfram.com/FrobeniusNorm.html - return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1])) + # The gradient of KL[p,q] is not defined when p==q. The culprit is + # linalg_ops.norm, i.e., we cannot use the commented out code. + # return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1])) + return math_ops.reduce_sum(math_ops.square(x), axis=[-2, -1]) # TODO(b/35041439): See also b/35040945. Remove this function once LinOp # supports something like: -- GitLab From b4aac5db53bb0f09a049d71b892ace2acc93ed9c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 17:33:34 -0700 Subject: [PATCH 393/573] Adds tests for eager execution to tf.metrics. Raises a ValueError for tf.metrics ops if they're executed in eager mode. PiperOrigin-RevId: 173476984 --- tensorflow/contrib/eager/python/BUILD | 1 + tensorflow/python/ops/metrics_impl.py | 146 ++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index bbbf72d632..340dca7e1a 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -35,6 +35,7 @@ cuda_py_test( additional_deps = [ ":tfe", "//tensorflow/python:array_ops", + "//tensorflow/python:metrics", "//tensorflow/python:math_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:platform_test", diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py index 10ff4be2dd..1858834f97 100644 --- a/tensorflow/python/ops/metrics_impl.py +++ b/tensorflow/python/ops/metrics_impl.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor @@ -323,7 +324,12 @@ def mean(values, weights=None, metrics_collections=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' + 'is enabled.') + with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) @@ -399,7 +405,12 @@ def accuracy(labels, predictions, weights=None, metrics_collections=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -626,7 +637,12 @@ def auc(labels, predictions, weights=None, num_thresholds=200, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.auc is not supported when eager execution ' + 'is enabled.') + with variable_scope.variable_scope( name, 'auc', (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': @@ -732,7 +748,12 @@ def mean_absolute_error(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' + 'when eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) absolute_errors = math_ops.abs(predictions - labels) @@ -783,7 +804,12 @@ def mean_cosine_distance(labels, predictions, dim, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' + 'eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) @@ -851,7 +877,12 @@ def mean_per_class_accuracy(labels, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' + 'when eager execution is enabled.') + with variable_scope.variable_scope(name, 'mean_accuracy', (predictions, labels, weights)): # Check if shape is compatible. @@ -934,7 +965,12 @@ def mean_iou(labels, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_iou is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope( name, 'mean_iou', (predictions, labels, weights)): # Check if shape is compatible. @@ -1027,7 +1063,12 @@ def mean_relative_error(labels, predictions, normalizer, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' + 'eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) @@ -1087,7 +1128,12 @@ def mean_squared_error(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' + 'eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) squared_error = math_ops.square(labels - predictions) @@ -1136,7 +1182,12 @@ def mean_tensor(values, weights=None, metrics_collections=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_tensor is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) total = _create_local('total_tensor', shape=values.get_shape()) @@ -1213,7 +1264,12 @@ def percentage_below(values, threshold, weights=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.percentage_below is not supported when ' + 'eager execution is enabled.') + is_below_threshold = math_ops.to_float(math_ops.less(values, threshold)) return mean(is_below_threshold, weights, @@ -1299,7 +1355,12 @@ def false_negatives(labels, predictions, weights=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_negatives is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope( name, 'false_negatives', (predictions, labels, weights)): @@ -1346,7 +1407,12 @@ def false_negatives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1392,7 +1458,12 @@ def false_positives(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_positives is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope( name, 'false_positives', (predictions, labels, weights)): @@ -1439,7 +1510,12 @@ def false_positives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1487,7 +1563,12 @@ def true_negatives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1533,7 +1614,12 @@ def true_positives(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.true_positives is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope( name, 'true_positives', (predictions, labels, weights)): @@ -1580,7 +1666,12 @@ def true_positives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1639,7 +1730,12 @@ def precision(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.precision is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope( name, 'precision', (predictions, labels, weights)): @@ -1721,7 +1817,12 @@ def precision_at_thresholds(labels, predictions, thresholds, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.precision_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'precision_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1787,7 +1888,12 @@ def recall(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.recall is not supported is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope( name, 'recall', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( @@ -2151,7 +2257,12 @@ def recall_at_k(labels, ValueError: If `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.recall_at_k is not ' + 'supported when eager execution is enabled.') + with ops.name_scope( name, _at_k_name('recall', k, class_id=class_id), (predictions, labels, weights)) as scope: @@ -2286,7 +2397,12 @@ def recall_at_thresholds(labels, predictions, thresholds, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.recall_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'recall_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -2354,7 +2470,12 @@ def root_mean_squared_error(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.root_mean_squared_error is not ' + 'supported when eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) mse, update_mse_op = mean_squared_error( @@ -2424,7 +2545,12 @@ def sensitivity_at_specificity( `weights` is not `None` and its shape doesn't match `predictions`, or if `specificity` is not between 0 and 1, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' + 'supported when eager execution is enabled.') + if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') @@ -2789,7 +2915,12 @@ def sparse_average_precision_at_k(labels, Raises: ValueError: if k is invalid. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' + 'supported when eager execution is enabled.') + if k < 1: raise ValueError('Invalid k=%s.' % k) with ops.name_scope( @@ -2953,7 +3084,12 @@ def precision_at_top_k(labels, ValueError: If `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.precision_at_top_k is not ' + 'supported when eager execution is enabled.') + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), (predictions_idx, labels, weights)) as scope: labels = _maybe_expand_labels(labels, predictions_idx) @@ -3048,7 +3184,12 @@ def sparse_precision_at_k(labels, ValueError: If `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' + 'supported when eager execution is enabled.') + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), (predictions, labels, weights)) as scope: _, top_k_idx = nn.top_k(predictions, k) @@ -3114,7 +3255,12 @@ def specificity_at_sensitivity( `weights` is not `None` and its shape doesn't match `predictions`, or if `sensitivity` is not between 0 and 1, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' + 'supported when eager execution is enabled.') + if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') -- GitLab From 5965a76ea72e266fba9b78adc94ec4ee71029ece Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Wed, 25 Oct 2017 17:44:18 -0700 Subject: [PATCH 394/573] [XLA] De-emphasize uninteresting nodes in the HLO graph dump. The hope is that this will make expensive / interesting ops easier to see. PiperOrigin-RevId: 173478095 --- .../compiler/xla/service/hlo_graph_dumper.cc | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index 20ec7dfe2f..b11b129c14 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -231,9 +231,9 @@ string HtmlLikeStringSanitize(tensorflow::StringPiece s) { // commutative, we also support them with param0 and param1 swapped. // // This is useful primarily for reduce and map nodes. These take a -// subcomputation which is almost always one of the four above, and pattern -// matching it to a short string lets us tell the user what the subcomputation -// is without drawing it as a graph. +// subcomputation which is almost always one of the above, and pattern matching +// it to a short string lets us tell the user what the subcomputation is without +// drawing it as a graph. optional MatchTrivialComputation(const HloComputation* computation) { if (computation->instruction_count() != 3) { return nullopt; @@ -788,7 +788,6 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kNegate: case HloOpcode::kPower: case HloOpcode::kRemainder: - case HloOpcode::kSelect: case HloOpcode::kShiftLeft: case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: @@ -799,21 +798,46 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kSubtract: case HloOpcode::kTanh: case HloOpcode::kRng: - case HloOpcode::kBroadcast: - case HloOpcode::kTranspose: + // De-emphasize scalar-shaped elementwise ops -- they're generally + // uninteresting. + if (ShapeUtil::IsEffectiveScalar(instr->shape())) { + return kWhite; + } return kYellow; case HloOpcode::kBitcast: case HloOpcode::kTuple: case HloOpcode::kTrace: case HloOpcode::kGetTupleElement: return kWhite; + case HloOpcode::kBroadcast: + // De-emphasize nodes which broadcast a scalar within a fusion node -- + // these are essentially free. + if (instr->IsFused() && + ShapeUtil::IsEffectiveScalar(instr->operand(0)->shape())) { + return kWhite; + } + return kGreen; case HloOpcode::kConcatenate: case HloOpcode::kCopy: case HloOpcode::kDynamicSlice: - case HloOpcode::kDynamicUpdateSlice: case HloOpcode::kPad: case HloOpcode::kReshape: case HloOpcode::kReverse: + case HloOpcode::kSelect: + case HloOpcode::kTranspose: + // De-emphasize scalar-shaped data movement ops and all data movement ops + // inside fusion nodes, both of which are essentially free. + if (ShapeUtil::IsEffectiveScalar(instr->shape()) || instr->IsFused()) { + return kWhite; + } + return kGreen; + case HloOpcode::kDynamicUpdateSlice: + // Unlike the data-movement ops above, dynamic-update-slice is not ~free + // inside of fusion nodes, so we de-emphasize it only if it's + // scalar-shaped. + if (ShapeUtil::IsEffectiveScalar(instr->shape())) { + return kWhite; + } return kGreen; case HloOpcode::kConvolution: case HloOpcode::kDot: -- GitLab From 03e2af6a819c5b45102c0f4b2a1a5f1f01c1a43e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Oct 2017 17:44:33 -0700 Subject: [PATCH 395/573] Update comment on xla::HloDCE to reflect that it does remove dead computations. PiperOrigin-RevId: 173478128 --- tensorflow/compiler/xla/service/hlo_dce.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_dce.h b/tensorflow/compiler/xla/service/hlo_dce.h index fca3fa0f58..4e244494d6 100644 --- a/tensorflow/compiler/xla/service/hlo_dce.h +++ b/tensorflow/compiler/xla/service/hlo_dce.h @@ -24,10 +24,15 @@ limitations under the License. namespace xla { -// HLO pass which removes all dead instructions from each computation in the -// module. An instruction is dead if it is not reachable from the root. This -// pass does not remove dead parameter instructions as parameter instructions -// cannot be deleted, nor does the pass remove dead computations. +// HLO pass which removes dead instructions from each computation in the module +// and removes dead computations from the module. +// +// An instruction is dead if it is not reachable from the root. A computation is +// dead if it is not the entry computation of the module and it is not reachable +// from the entry computation. +// +// This pass does not remove dead parameter instructions, as parameter +// instructions cannot be deleted. class HloDCE : public HloPassInterface { public: ~HloDCE() override {} -- GitLab From 6149fecbdba96fea5460915cf2fad5ac163de091 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 25 Oct 2017 17:45:38 -0700 Subject: [PATCH 396/573] Internal Change PiperOrigin-RevId: 173478239 --- tensorflow/workspace.bzl | 10 ++ third_party/flatbuffers/BUILD | 15 ++ third_party/flatbuffers/build_defs.bzl | 196 ++++++++++++++++++++++ third_party/flatbuffers/flatbuffers.BUILD | 127 ++++++++++++++ 4 files changed, 348 insertions(+) create mode 100644 third_party/flatbuffers/BUILD create mode 100644 third_party/flatbuffers/build_defs.bzl create mode 100644 third_party/flatbuffers/flatbuffers.BUILD diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index f5006ad55d..b9651a92f7 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -780,3 +780,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")), ) + + native.new_http_archive( + name = "flatbuffers", + build_file = "third_party/flatbuffers/flatbuffers.BUILD", + strip_prefix = "flatbuffers-971a68110e4fc1bace10fcb6deeb189e7e1a34ce", + sha256 = "874088d2ee0d9f8524191f77209556415f03dd44e156276edf19e5b90ceb5f55", + urls = [ + "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz", + ], + ) diff --git a/third_party/flatbuffers/BUILD b/third_party/flatbuffers/BUILD new file mode 100644 index 0000000000..fbdf19f205 --- /dev/null +++ b/third_party/flatbuffers/BUILD @@ -0,0 +1,15 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/third_party/flatbuffers/build_defs.bzl b/third_party/flatbuffers/build_defs.bzl new file mode 100644 index 0000000000..ae8d7feebe --- /dev/null +++ b/third_party/flatbuffers/build_defs.bzl @@ -0,0 +1,196 @@ +# Description: +# BUILD rules for generating flatbuffer files. + +flatc_path = "@flatbuffers//:flatc" + +DEFAULT_FLATC_ARGS = [ + "--no-union-value-namespacing", + "--gen-object-api", +] + +def flatbuffer_library_public(name, + srcs, + outs, + language_flag, + out_prefix="", + includes=[], + include_paths=[], + flatc_args=DEFAULT_FLATC_ARGS, + reflection_name="", + reflection_visiblity=None, + output_to_bindir=False): + '''Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler. + + Args: + name: Rule name. + srcs: Source .fbs files. Sent in order to the compiler. + outs: Output files from flatc. + language_flag: Target language flag. One of [-c, -j, -js]. + out_prefix: Prepend this path to the front of all generated files except on + single source targets. Usually is a directory name. + includes: Optional, list of filegroups of schemas that the srcs depend on. + include_paths: Optional, list of paths the includes files can be found in. + flatc_args: Optional, list of additional arguments to pass to flatc. + reflection_name: Optional, if set this will generate the flatbuffer + reflection binaries for the schemas. + reflection_visiblity: The visibility of the generated reflection Fileset. + output_to_bindir: Passed to genrule for output to bin directory. + Outs: + filegroup(name): all generated source files. + Fileset([reflection_name]): (Optional) all generated reflection binaries. + ''' + include_paths_cmd = ["-I %s" % (s) for s in include_paths] + # '$(@D)' when given a single source target will give the appropriate + # directory. Appending 'out_prefix' is only necessary when given a build + # target with multiple sources. + output_directory = ( + ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)")) + genrule_cmd = " ".join([ + "for f in $(SRCS); do", + "$(location %s)" % (flatc_path), + " ".join(flatc_args), + " ".join(include_paths_cmd), + language_flag, + output_directory, + "$$f;", + "done", + ]) + native.genrule( + name=name, + srcs=srcs, + outs=outs, + output_to_bindir=output_to_bindir, + tools=includes + [flatc_path,], + cmd=genrule_cmd, + message="Generating flatbuffer files for %s:" % (name),) + if reflection_name: + reflection_genrule_cmd = " ".join([ + "for f in $(SRCS); do", + "$(location %s)" % (flatc_path), + "-b --schema", + " ".join(flatc_args), + " ".join(include_paths_cmd), + language_flag, + output_directory, + "$$f;", + "done", + ]) + reflection_outs = [ + (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs + ] + native.genrule( + name= "%s_srcs" % reflection_name, + srcs=srcs, + outs=reflection_outs, + output_to_bindir=output_to_bindir, + tools=includes + [flatc_path,], + cmd=reflection_genrule_cmd, + message="Generating flatbuffer reflection binary for %s:" % (name),) + native.Fileset( + name=reflection_name, + out="%s_out" % reflection_name, + entries=[ + native.FilesetEntry(files=reflection_outs), + ], + visibility=reflection_visiblity + ) + + +def flatbuffer_cc_library(name, srcs, srcs_filegroup_name="", + out_prefix="", includes=[], include_paths=[], + flatc_args=DEFAULT_FLATC_ARGS, + visibility=None, srcs_filegroup_visibility=None, + gen_reflections=False): + '''A cc_library with the generated reader/writers for the given flatbuffer definitions. + + Args: + name: Rule name. + srcs: Source .fbs files. Sent in order to the compiler. + srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this + filegroup into the `includes` parameter of any other + flatbuffer_cc_library that depends on this one's schemas. + out_prefix: Prepend this path to the front of all generated files. Usually + is a directory name. + includes: Optional, list of filegroups of schemas that the srcs depend on. + ** SEE REMARKS BELOW ** + include_paths: Optional, list of paths the includes files can be found in. + flatc_args: Optional list of additional arguments to pass to flatc + (e.g. --gen-mutable). + visibility: The visibility of the generated cc_library. By default, use the + default visibility of the project. + srcs_filegroup_visibility: The visibility of the generated srcs filegroup. + By default, use the value of the visibility parameter above. + gen_reflections: Optional, if true this will generate the flatbuffer + reflection binaries for the schemas. + Outs: + filegroup([name]_srcs): all generated .h files. + filegroup(srcs_filegroup_name if specified, or [name]_includes if not): + Other flatbuffer_cc_library's can pass this in for their `includes` + parameter, if they depend on the schemas in this library. + Fileset([name]_reflection): (Optional) all generated reflection binaries. + cc_library([name]): library with sources and flatbuffers deps. + + Remarks: + ** Because the genrule used to call flatc does not have any trivial way of + computing the output list of files transitively generated by includes and + --gen-includes (the default) being defined for flatc, the --gen-includes + flag will not work as expected. The way around this is to add a dependency + to the flatbuffer_cc_library defined alongside the flatc included Fileset. + For example you might define: + + flatbuffer_cc_library( + name = "my_fbs", + srcs = [ "schemas/foo.fbs" ], + includes = [ "//third_party/bazz:bazz_fbs_includes" ], + ) + + In which foo.fbs includes a few files from the Fileset defined at + //third_party/bazz:bazz_fbs_includes. When compiling the library that + includes foo_generated.h, and therefore has my_fbs as a dependency, it + will fail to find any of the bazz *_generated.h files unless you also + add bazz's flatbuffer_cc_library to your own dependency list, e.g.: + + cc_library( + name = "my_lib", + deps = [ + ":my_fbs", + "//third_party/bazz:bazz_fbs" + ], + ) + + Happy dependent Flatbuffering! + ''' + output_headers = [ + (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs + ] + reflection_name = "%s_reflection" % name if gen_reflections else "" + + flatbuffer_library_public(name="%s_srcs" % (name), + srcs=srcs, + outs=output_headers, + language_flag="-c", + out_prefix=out_prefix, + includes=includes, + include_paths=include_paths, + flatc_args=flatc_args, + reflection_name=reflection_name, + reflection_visiblity=visibility,) + native.cc_library(name=name, + hdrs=output_headers, + srcs=output_headers, + features=[ + "-parse_headers", + ], + deps=[ + "@flatbuffers//:runtime_cc", + ], + includes=["."], + linkstatic=1, + visibility=visibility) + + # A filegroup for the `srcs`. That is, all the schema files for this + # Flatbuffer set. + native.filegroup( + name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name), + srcs = srcs, + visibility=srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility) diff --git a/third_party/flatbuffers/flatbuffers.BUILD b/third_party/flatbuffers/flatbuffers.BUILD new file mode 100644 index 0000000000..a426db0c50 --- /dev/null +++ b/third_party/flatbuffers/flatbuffers.BUILD @@ -0,0 +1,127 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +FLATBUFFERS_COPTS = [ + "-fexceptions", + "-Wno-implicit-fallthrough", +] + +# Public flatc library to compile flatbuffer files at runtime. +cc_library( + name = "flatbuffers", + srcs = [ + "include/flatbuffers/code_generators.h", + "include/flatbuffers/reflection_generated.h", + "src/code_generators.cpp", + "src/idl_gen_fbs.cpp", + "src/idl_gen_general.cpp", + "src/idl_gen_text.cpp", + "src/idl_parser.cpp", + "src/reflection.cpp", + "src/util.cpp", + ], + hdrs = [ + "include/flatbuffers/base.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/flexbuffers.h", + "include/flatbuffers/hash.h", + "include/flatbuffers/idl.h", + "include/flatbuffers/reflection.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + ], + copts = FLATBUFFERS_COPTS, + includes = ["include/"], +) + +# Public flatc compiler library. +cc_library( + name = "flatc_library", + srcs = [ + "grpc/src/compiler/config.h", + "grpc/src/compiler/go_generator.h", + "grpc/src/compiler/schema_interface.h", + "include/flatbuffers/base.h", + "include/flatbuffers/code_generators.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/flatc.h", + "include/flatbuffers/flexbuffers.h", + "include/flatbuffers/hash.h", + "include/flatbuffers/idl.h", + "include/flatbuffers/reflection.h", + "include/flatbuffers/reflection_generated.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + "src/code_generators.cpp", + "src/flatc.cpp", + "src/idl_gen_fbs.cpp", + "src/idl_parser.cpp", + "src/reflection.cpp", + "src/util.cpp", + ], + hdrs = [ + "include/flatbuffers/base.h", + "include/flatbuffers/code_generators.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/flatc.h", + "include/flatbuffers/idl.h", + "include/flatbuffers/reflection.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + ], + copts = FLATBUFFERS_COPTS, + includes = [ + "grpc/", + "include/", + ], +) + +# Public flatc compiler. +cc_binary( + name = "flatc", + srcs = [ + "grpc/src/compiler/cpp_generator.cc", + "grpc/src/compiler/cpp_generator.h", + "grpc/src/compiler/go_generator.cc", + "grpc/src/compiler/go_generator.h", + "grpc/src/compiler/schema_interface.h", + "src/flatc_main.cpp", + "src/idl_gen_cpp.cpp", + "src/idl_gen_general.cpp", + "src/idl_gen_go.cpp", + "src/idl_gen_grpc.cpp", + "src/idl_gen_js.cpp", + "src/idl_gen_json_schema.cpp", + "src/idl_gen_php.cpp", + "src/idl_gen_python.cpp", + "src/idl_gen_text.cpp", + ], + copts = FLATBUFFERS_COPTS, + includes = [ + "grpc/", + "include/", + ], + deps = [ + ":flatc_library", + ], +) + +filegroup( + name = "runtime_cc_srcs", + srcs = [ + "include/flatbuffers/base.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + ], +) + +cc_library( + name = "runtime_cc", + hdrs = ["runtime_cc_srcs"], + includes = ["include"], + linkstatic = 1, +) -- GitLab From ff7b9a6c496823c1bffdd0d74bf68aafacb8caca Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 25 Oct 2017 18:46:35 -0700 Subject: [PATCH 397/573] Adding summaries to the resnet example. Also utilities to use summaries in graph mode. PiperOrigin-RevId: 173483424 --- tensorflow/contrib/BUILD | 2 +- tensorflow/contrib/__init__.py | 1 + tensorflow/contrib/cmake/tf_core_ops.cmake | 1 + tensorflow/contrib/cmake/tf_python.cmake | 3 + tensorflow/contrib/summary/BUILD | 25 ++++++ tensorflow/contrib/summary/summary.py | 39 +++++++++ tensorflow/contrib/summary/summary_ops.py | 82 ++++++++++++++----- .../contrib/summary/summary_ops_test.py | 29 ++----- .../contrib/summary/summary_test_util.py | 41 ++++++++++ tensorflow/tools/pip_package/BUILD | 1 + 10 files changed, 182 insertions(+), 42 deletions(-) create mode 100644 tensorflow/contrib/summary/summary.py create mode 100644 tensorflow/contrib/summary/summary_test_util.py diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 3d580fae14..ee3dd5079e 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -80,7 +80,7 @@ py_library( "//tensorflow/contrib/staging", "//tensorflow/contrib/stat_summarizer:stat_summarizer_py", "//tensorflow/contrib/stateless", - "//tensorflow/contrib/summary:summary_ops", + "//tensorflow/contrib/summary:summary", "//tensorflow/contrib/tensor_forest:init_py", "//tensorflow/contrib/tensorboard", "//tensorflow/contrib/testing:testing_py", diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index bf921808aa..76a629663d 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -80,6 +80,7 @@ from tensorflow.contrib import util from tensorflow.contrib.ndlstm import python as ndlstm from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph from tensorflow.contrib.specs import python as specs +from tensorflow.contrib.summary import summary from tensorflow.python.util.lazy_loader import LazyLoader ffmpeg = LazyLoader("ffmpeg", diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake index dc9973917e..97bec81e66 100644 --- a/tensorflow/contrib/cmake/tf_core_ops.cmake +++ b/tensorflow/contrib/cmake/tf_core_ops.cmake @@ -43,6 +43,7 @@ set(tf_op_lib_names "state_ops" "stateless_random_ops" "string_ops" + "summary_ops" "training_ops" ) diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 8ddfb59595..a3ed19977f 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -640,6 +640,7 @@ add_python_module("tensorflow/contrib/reduce_slice_ops/ops") add_python_module("tensorflow/contrib/reduce_slice_ops/python") add_python_module("tensorflow/contrib/reduce_slice_ops/python/kernel_tests") add_python_module("tensorflow/contrib/reduce_slice_ops/python/ops") +add_python_module("tensorflow/contrib/summary") # Generate the tensorflow.python.platform.build_info module. set(BUILD_INFO_PY "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/platform/build_info.py") @@ -812,6 +813,8 @@ GENERATE_PYTHON_OP_LIB("stateless_random_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/stateless/gen_stateless_random_ops.py) GENERATE_PYTHON_OP_LIB("debug_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/debug/ops/gen_debug_ops.py) +GENERATE_PYTHON_OP_LIB("summary_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/summary/gen_summary_ops.py) add_custom_target(tf_python_ops SOURCES ${tf_python_ops_generated_files} ${PYTHON_PROTO_GENFILES}) add_dependencies(tf_python_ops tf_python_op_gen_main) diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD index bcb2d74b4a..8cb5c3f381 100644 --- a/tensorflow/contrib/summary/BUILD +++ b/tensorflow/contrib/summary/BUILD @@ -25,6 +25,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":summary_ops", + ":summary_test_util", "//tensorflow/core:protos_all_py", "//tensorflow/python:framework_test_lib", "//tensorflow/python:lib", @@ -52,6 +53,16 @@ py_library( ], ) +py_library( + name = "summary", + srcs = ["summary.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + ":summary_ops", + ], +) + filegroup( name = "all_files", srcs = glob( @@ -63,3 +74,17 @@ filegroup( ), visibility = ["//tensorflow:__subpackages__"], ) + +# NOTE: target cannot be testonly because it needs to be in the pip +# package. Sigh. +py_library( + name = "summary_test_util", + srcs = ["summary_test_util.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + "//tensorflow/core:protos_all_py", + "//tensorflow/python:lib", + "//tensorflow/python:platform", + ], +) diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py new file mode 100644 index 0000000000..89031caadc --- /dev/null +++ b/tensorflow/contrib/summary/summary.py @@ -0,0 +1,39 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Contrib summary package. + +The operations in this package are safe to use with eager execution turned or on +off. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import +from tensorflow.contrib.summary.summary_ops import all_summary_ops +from tensorflow.contrib.summary.summary_ops import always_record_summaries +from tensorflow.contrib.summary.summary_ops import audio +from tensorflow.contrib.summary.summary_ops import create_summary_file_writer +from tensorflow.contrib.summary.summary_ops import generic +from tensorflow.contrib.summary.summary_ops import histogram +from tensorflow.contrib.summary.summary_ops import image +from tensorflow.contrib.summary.summary_ops import never_record_summaries +from tensorflow.contrib.summary.summary_ops import record_summaries_every_n_global_steps +from tensorflow.contrib.summary.summary_ops import scalar +from tensorflow.contrib.summary.summary_ops import should_record_summaries +from tensorflow.contrib.summary.summary_ops import summary_writer_initializer_op diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index 30a9398ee5..b32b093675 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -25,6 +25,8 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.layers import utils +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import summary_op_util from tensorflow.python.training import training_util from tensorflow.python.util import tf_contextlib @@ -33,6 +35,9 @@ from tensorflow.python.util import tf_contextlib # Tensor. If this tensor is True the summary ops will record summaries. _SHOULD_RECORD_SUMMARIES_NAME = "ShouldRecordSummaries" +_SUMMARY_COLLECTION_NAME = "_SUMMARY_V2" +_SUMMARY_WRITER_INIT_COLLECTION_NAME = "_SUMMARY_WRITER_V2" + def should_record_summaries(): """Returns boolean Tensor which is true if summaries should be recorded.""" @@ -78,10 +83,15 @@ def never_record_summaries(): class SummaryWriter(object): + """Encapsulates a summary writer.""" def __init__(self, resource): self._resource = resource + def __del__(self): + if context.in_eager_mode(): + resource_variable_ops.destroy_resource_op(self._resource) + def set_as_default(self): context.context().summary_writer_resource = self._resource @@ -90,6 +100,9 @@ class SummaryWriter(object): old = context.context().summary_writer_resource context.context().summary_writer_resource = self._resource yield + # Flushes the summary writer in eager mode or in graph functions, but not in + # legacy graph mode (you're on your own there). + gen_summary_ops.flush_summary_writer(self._resource) context.context().summary_writer_resource = old @@ -108,14 +121,33 @@ def create_summary_file_writer(logdir, resource = gen_summary_ops.summary_writer(shared_name=name) # TODO(apassos) ensure the initialization op runs when in graph mode; consider # calling session.run here. - gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, - flush_secs, filename_suffix) + ops.add_to_collection( + _SUMMARY_WRITER_INIT_COLLECTION_NAME, + gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, + flush_secs, filename_suffix)) return SummaryWriter(resource) def _nothing(): """Convenient else branch for when summaries do not record.""" - return False + return constant_op.constant(False) + + +def all_summary_ops(): + """Graph-mode only. Returns all summary ops.""" + if context.in_eager_mode(): + raise RuntimeError( + "tf.contrib.summary.all_summary_ops is only supported in graph mode.") + return ops.get_collection(_SUMMARY_COLLECTION_NAME) + + +def summary_writer_initializer_op(): + """Graph-mode only. Returns the list of ops to create all summary writers.""" + if context.in_eager_mode(): + raise RuntimeError( + "tf.contrib.summary.summary_writer_initializer_op is only " + "supported in graph mode.") + return ops.get_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME) def summary_writer_function(name, tensor, function, family=None): @@ -133,20 +165,25 @@ def summary_writer_function(name, tensor, function, family=None): def record(): with summary_op_util.summary_scope( name, family, values=[tensor]) as (tag, scope): - function(tag, scope) - return True + with ops.control_dependencies([function(tag, scope)]): + return constant_op.constant(True) - return utils.smart_cond( - should_record_summaries(), record, _nothing, name="") + with ops.device("cpu:0"): + op = utils.smart_cond( + should_record_summaries(), record, _nothing, name="") + ops.add_to_collection(_SUMMARY_COLLECTION_NAME, op) + return op def generic(name, tensor, metadata, family=None): """Writes a tensor summary if possible.""" def function(tag, scope): - gen_summary_ops.write_summary(context.context().summary_writer_resource, - training_util.get_global_step(), tensor, - tag, metadata, name=scope) + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_summary( + context.context().summary_writer_resource, + training_util.get_global_step(), array_ops.identity(tensor), + tag, metadata, name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -154,9 +191,11 @@ def scalar(name, tensor, family=None): """Writes a scalar summary if possible.""" def function(tag, scope): - gen_summary_ops.write_scalar_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_scalar_summary( context.context().summary_writer_resource, - training_util.get_global_step(), tag, tensor, name=scope) + training_util.get_global_step(), tag, array_ops.identity(tensor), + name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -165,9 +204,11 @@ def histogram(name, tensor, family=None): """Writes a histogram summary if possible.""" def function(tag, scope): - gen_summary_ops.write_histogram_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_histogram_summary( context.context().summary_writer_resource, - training_util.get_global_step(), tag, tensor, name=scope) + training_util.get_global_step(), tag, array_ops.identity(tensor), + name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -178,10 +219,12 @@ def image(name, tensor, bad_color=None, max_images=3, family=None): def function(tag, scope): if bad_color is None: bad_color_ = constant_op.constant([255, 0, 0, 255], dtype=dtypes.uint8) - gen_summary_ops.write_image_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_image_summary( context.context().summary_writer_resource, - training_util.get_global_step(), tag, tensor, bad_color_, max_images, - name=scope) + training_util.get_global_step(), tag, array_ops.identity(tensor), + bad_color_, + max_images, name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -190,11 +233,12 @@ def audio(name, tensor, sample_rate, max_outputs, family=None): """Writes an audio summary if possible.""" def function(tag, scope): - gen_summary_ops.write_audio_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_audio_summary( context.context().summary_writer_resource, training_util.get_global_step(), tag, - tensor, + array_ops.identity(tensor), sample_rate=sample_rate, max_outputs=max_outputs, name=scope) diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index 405a92a726..de7ae6ec27 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -17,16 +17,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import tempfile from tensorflow.contrib.summary import summary_ops -from tensorflow.core.util import event_pb2 +from tensorflow.contrib.summary import summary_test_util from tensorflow.python.eager import function from tensorflow.python.eager import test from tensorflow.python.framework import errors from tensorflow.python.framework import test_util -from tensorflow.python.lib.io import tf_record from tensorflow.python.platform import gfile from tensorflow.python.training import training_util @@ -71,16 +69,9 @@ class TargetTest(test_util.TensorFlowTestCase): summary_ops.scalar('scalar', 2.0) write() - - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list( - tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].simple_value, 2.0) + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 2.0) def testSummaryName(self): training_util.get_or_create_global_step() @@ -91,15 +82,9 @@ class TargetTest(test_util.TensorFlowTestCase): summary_ops.scalar('scalar', 2.0) - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list( - tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].tag, 'scalar') + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].tag, 'scalar') if __name__ == '__main__': diff --git a/tensorflow/contrib/summary/summary_test_util.py b/tensorflow/contrib/summary/summary_test_util.py new file mode 100644 index 0000000000..37b546d3ab --- /dev/null +++ b/tensorflow/contrib/summary/summary_test_util.py @@ -0,0 +1,41 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Utilities to test summaries.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from tensorflow.core.util import event_pb2 +from tensorflow.python.lib.io import tf_record +from tensorflow.python.platform import gfile + + +def events_from_file(logdir): + """Returns all events in the single eventfile in logdir.""" + assert gfile.Exists(logdir) + files = gfile.ListDirectory(logdir) + assert len(files) == 1, "Found more than one file in logdir: %s" % files + records = list( + tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) + result = [] + for r in records: + event = event_pb2.Event() + event.ParseFromString(r) + result.append(event) + return result diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 3c4e1b66bc..579c51ab3a 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -85,6 +85,7 @@ py_binary( "//tensorflow/python:spectral_ops_test_util", "//tensorflow/python/tools:tools_pip", "//tensorflow/python/eager:eager_pip", + "//tensorflow/contrib/summary:summary_test_util", # These targets don't build on Windows yet. Exclude them for now. # "//tensorflow/contrib/ndlstm", # "//tensorflow/contrib/slim", -- GitLab From 65616777d73913346ec446df80b6d7aa64e54b24 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Wed, 25 Oct 2017 20:46:06 -0700 Subject: [PATCH 398/573] Update the ReferenceResolver to output html links. PiperOrigin-RevId: 173491069 --- tensorflow/tools/docs/parser.py | 70 ++++++++++++++++++++-------- tensorflow/tools/docs/parser_test.py | 31 ++++++------ 2 files changed, 69 insertions(+), 32 deletions(-) diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py index 1015103077..3db164c2b5 100644 --- a/tensorflow/tools/docs/parser.py +++ b/tensorflow/tools/docs/parser.py @@ -107,23 +107,40 @@ def _get_raw_docstring(py_object): # A regular expression for capturing a @{symbol} reference. -SYMBOL_REFERENCE_RE = re.compile(r'@\{([^}]+)\}') +SYMBOL_REFERENCE_RE = re.compile( + r""" + # Start with a literal "@{". + @\{ + # Group at least 1 symbol: not "}" or "\n". + ([^}\n]+) + # Followed by a closing "}" + \} + """, + flags=re.VERBOSE) class ReferenceResolver(object): """Class for replacing @{...} references with Markdown links. - Args: - duplicate_of: A map from duplicate names to preferred names of API - symbols. - doc_index: A `dict` mapping symbol name strings to objects with `url` - and `title` fields. Used to resolve @{$doc} references in docstrings. - index: A map from all full names to python objects. - py_module_names: A list of string names of Python modules. + Attributes: + current_doc_full_name: A string (or None) indicating the name of the + document currently being processed, so errors can reference the broken + doc. """ def __init__(self, duplicate_of, doc_index, is_class, is_module, py_module_names): + """Initializes a Reference Resolver. + + Args: + duplicate_of: A map from duplicate names to preferred names of API + symbols. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. + is_class: A map from full names to bool for each symbol. + is_module: A map from full names to bool for each symbol. + py_module_names: A list of string names of Python modules. + """ self._duplicate_of = duplicate_of self._doc_index = doc_index self._is_class = is_class @@ -249,11 +266,19 @@ class ReferenceResolver(object): Returns: A markdown link to the documentation page of `ref_full_name`. """ - link = self.reference_to_url(ref_full_name, relative_path_to_root) + url = self.reference_to_url(ref_full_name, relative_path_to_root) + if code_ref: - return '[`%s`](%s)' % (link_text, link) + link_text = link_text.join(['', '']) else: - return '[%s](%s)' % (link_text, link) + link_text = self._link_text_to_html(link_text) + + return '{}'.format(url, link_text) + + @staticmethod + def _link_text_to_html(link_text): + code_re = '`(.*?)`' + return re.sub(code_re, r'\1', link_text) def py_master_name(self, full_name): """Return the master name for a Python symbol name.""" @@ -322,13 +347,13 @@ class ReferenceResolver(object): # Handle different types of references. if string.startswith('$'): # Doc reference - return self._doc_link( - string, link_text, manual_link_text, relative_path_to_root) + return self._doc_link(string, link_text, manual_link_text, + relative_path_to_root) elif string.startswith('tensorflow::'): # C++ symbol - return self._cc_link( - string, link_text, manual_link_text, relative_path_to_root) + return self._cc_link(string, link_text, manual_link_text, + relative_path_to_root) else: is_python = False @@ -337,8 +362,11 @@ class ReferenceResolver(object): is_python = True break if is_python: # Python symbol - return self.python_link(link_text, string, relative_path_to_root, - code_ref=not manual_link_text) + return self.python_link( + link_text, + string, + relative_path_to_root, + code_ref=not manual_link_text) # Error! self.add_error('Did not understand "%s"' % match.group(0)) @@ -361,7 +389,9 @@ class ReferenceResolver(object): if not manual_link_text: link_text = self._doc_index[string].title url = os.path.normpath(os.path.join( relative_path_to_root, '../..', self._doc_index[string].url)) - return '[%s](%s%s)' % (link_text, url, hash_tag) + link_text = self._link_text_to_html(link_text) + return '{}'.format(url, hash_tag, link_text) + return self._doc_missing(string, hash_tag, link_text, manual_link_text, relative_path_to_root) @@ -392,7 +422,9 @@ class ReferenceResolver(object): # to api_docs/cc, and then add ret. cc_relative_path = os.path.normpath(os.path.join( relative_path_to_root, '../cc', ret)) - return '[`%s`](%s)' % (link_text, cc_relative_path) + + return '{}'.format(cc_relative_path, + link_text) # TODO(aselle): Collect these into a big list for all modules and functions diff --git a/tensorflow/tools/docs/parser_test.py b/tensorflow/tools/docs/parser_test.py index 3b74a13f08..8a0e9af521 100644 --- a/tensorflow/tools/docs/parser_test.py +++ b/tensorflow/tools/docs/parser_test.py @@ -75,8 +75,9 @@ class ParserTest(googletest.TestCase): def foo(self): pass - string = ('A @{tf.reference}, another @{tf.reference}, ' - 'a member @{tf.reference.foo}, and a @{tf.third}.') + string = ( + 'A @{tf.reference}, another @{tf.reference}, a member ' + '@{tf.reference.foo}, and a @{tf.third$link `text` with `code` in it}.') duplicate_of = {'tf.third': 'tf.fourth'} index = {'tf.reference': HasOneMember, 'tf.reference.foo': HasOneMember.foo, @@ -89,12 +90,15 @@ class ParserTest(googletest.TestCase): visitor=visitor, doc_index={}, py_module_names=['tf']) result = reference_resolver.replace_references(string, '../..') - self.assertEqual( - 'A [`tf.reference`](../../tf/reference.md), another ' - '[`tf.reference`](../../tf/reference.md), ' - 'a member [`tf.reference.foo`](../../tf/reference.md#foo), ' - 'and a [`tf.third`](../../tf/fourth.md).', - result) + self.assertEqual('A ' + 'tf.reference, ' + 'another ' + 'tf.reference, ' + 'a member ' + 'tf.reference.foo, ' + 'and a link ' + 'text with ' + 'code in it.', result) def test_doc_replace_references(self): string = '@{$doc1} @{$doc1#abc} @{$doc1$link} @{$doc1#def$zelda} @{$do/c2}' @@ -114,10 +118,11 @@ class ParserTest(googletest.TestCase): reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index=doc_index, py_module_names=['tf']) result = reference_resolver.replace_references(string, 'python') - self.assertEqual( - '[Title1](../URL1) [Title1](../URL1#abc) [link](../URL1) ' - '[zelda](../URL1#def) [Two words](../somewhere/else)', - result) + self.assertEqual('Title1 ' + 'Title1 ' + 'link ' + 'zelda ' + 'Two words', result) def test_docs_for_class(self): @@ -389,7 +394,7 @@ class ParserTest(googletest.TestCase): self.assertIn('TestModule.test_function', docs) # Leading backtick to make sure it's included top-level. # This depends on formatting, but should be stable. - self.assertIn('`test_function', docs) + self.assertIn('test_function', docs) def test_argspec_for_functools_partial(self): -- GitLab From 62f62bfe9f0ffba8ae406dcb1b6c0165d38fe633 Mon Sep 17 00:00:00 2001 From: James Qin Date: Wed, 25 Oct 2017 21:38:45 -0700 Subject: [PATCH 399/573] Switch tf.contrib.cudnn_rnn.CudnnXXX to point to layer APIs instead of op wrappers PiperOrigin-RevId: 173494053 --- tensorflow/contrib/cudnn_rnn/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/cudnn_rnn/__init__.py b/tensorflow/contrib/cudnn_rnn/__init__.py index 87ba834770..bc44562b50 100644 --- a/tensorflow/contrib/cudnn_rnn/__init__.py +++ b/tensorflow/contrib/cudnn_rnn/__init__.py @@ -30,13 +30,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnGRU +from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnLSTM +from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNRelu +from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNTanh from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleGRUCell from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleLSTMCell -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRU from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRUSaveable -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTM from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTMSaveable -from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNRelu from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNReluSaveable from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNTanhSaveable -- GitLab From 09c5bf350ac634922328bc752e9250cf24966478 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Wed, 25 Oct 2017 23:03:44 -0700 Subject: [PATCH 400/573] Add support for PRED and S64 types to MakeFakeLiteral. Don't uniquify names when creating and HLO module from a proto. This preserves instruction names across serialization/deserialization. PiperOrigin-RevId: 173498734 --- tensorflow/compiler/xla/client/lib/testing.cc | 18 +++++ .../compiler/xla/service/hlo_computation.h | 8 +- tensorflow/compiler/xla/service/hlo_module.cc | 80 ++++++++++++++----- tensorflow/compiler/xla/service/hlo_module.h | 3 +- 4 files changed, 80 insertions(+), 29 deletions(-) diff --git a/tensorflow/compiler/xla/client/lib/testing.cc b/tensorflow/compiler/xla/client/lib/testing.cc index 482d53cf33..e6645e4941 100644 --- a/tensorflow/compiler/xla/client/lib/testing.cc +++ b/tensorflow/compiler/xla/client/lib/testing.cc @@ -79,6 +79,24 @@ StatusOr> MakeFakeLiteral(const Shape& shape) { })); break; } + case S64: { + std::uniform_int_distribution generator( + std::numeric_limits::lowest(), + std::numeric_limits::max()); + TF_CHECK_OK(literal->Populate( + [&](tensorflow::gtl::ArraySlice /*indices*/) { + return generator(engine); + })); + break; + } + case PRED: { + std::uniform_int_distribution generator(0, 1); + TF_CHECK_OK(literal->Populate( + [&](tensorflow::gtl::ArraySlice /*indices*/) { + return generator(engine); + })); + break; + } default: return Unimplemented("Unsupported type for fake literal generation: %s", ShapeUtil::HumanString(shape).c_str()); diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h index 3515a6b5df..f4edd17501 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.h +++ b/tensorflow/compiler/xla/service/hlo_computation.h @@ -312,8 +312,7 @@ class HloComputation { explicit HloComputation( const string& name, int parameter_count, std::vector>* instructions, - HloInstruction* root_instruction, - HloInstruction* fusion_instruction = nullptr); + HloInstruction* root_instruction, HloInstruction* fusion_instruction); // Internal helper for adding instructions. HloInstruction* AddInstructionInternal( @@ -359,11 +358,6 @@ class HloComputation { std::vector param_instructions_; - // Unique name generator for instruction identifiers. Instruction names should - // be unique per computation and this is enforced when instructions are added - // to the computation. - NameUniquer instruction_name_uniquer_; - TF_DISALLOW_COPY_AND_ASSIGN(HloComputation); }; diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index f7990fa789..4779ec7760 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -45,10 +45,37 @@ HloModule::HloModule(const string& name, const HloModuleConfig& config) : name_(name), config_(config) {} HloComputation* HloModule::AddComputationInternal( - std::unique_ptr computation) { - computation->UniquifyName(&computation_name_uniquer_); + std::unique_ptr computation, bool is_entry, + bool uniquify_names) { + if (is_entry) { + CHECK_EQ(nullptr, entry_computation_); + entry_computation_ = computation.get(); + + // If the module configuration has no entry layout computation set, create a + // default one based on the program shape. + if (!config_.has_entry_computation_layout()) { + config_.SetDefaultComputationLayout( + entry_computation_->ComputeProgramShape()); + } + } + + if (uniquify_names) { + computation->UniquifyName(&computation_name_uniquer_); + for (auto* instruction : computation->instructions()) { + instruction->UniquifyName(&instruction_name_uniquer_); + } + } else { + // Don't uniquify the names of the computation or instruction, but we must + // run the names through the uniquifiers to prevent future name collisions + // for computations and instructions created later. + computation_name_uniquer_.GetUniqueName(computation->name()); + for (auto* instruction : computation->instructions()) { + instruction_name_uniquer_.GetUniqueName(instruction->name()); + } + } + + // Pick unique IDs for each instruction. for (auto* instruction : computation->instructions()) { - instruction->UniquifyName(&instruction_name_uniquer_); instruction->SetUniqueId(NewUniqueInstructionId()); } computation->set_parent(this); @@ -58,16 +85,8 @@ HloComputation* HloModule::AddComputationInternal( HloComputation* HloModule::AddEntryComputation( std::unique_ptr computation) { - CHECK_EQ(nullptr, entry_computation_); - entry_computation_ = computation.get(); - - // If the module configuration has no entry layout computation set, create a - // default one based on the program shape. - if (!config_.has_entry_computation_layout()) { - config_.SetDefaultComputationLayout( - entry_computation_->ComputeProgramShape()); - } - return AddComputationInternal(std::move(computation)); + return AddComputationInternal(std::move(computation), /*is_entry=*/true, + /*uniquify_names=*/true); } Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) { @@ -83,7 +102,8 @@ Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) { HloComputation* HloModule::AddEmbeddedComputation( std::unique_ptr computation) { - return AddComputationInternal(std::move(computation)); + return AddComputationInternal(std::move(computation), /*is_entry=*/false, + /*uniquify_names=*/true); } void HloModule::ReplaceComputations( @@ -199,16 +219,34 @@ StatusOr> HloModule::CreateFromProto( CHECK_NE(computation.get(), nullptr); TF_RET_CHECK(!ContainsKey(computation_map, computation->name())); string computation_name = computation->name(); - if (proto.entry_computation_name() == computation_name) { - computation_map[computation_name] = - module->AddEntryComputation(std::move(computation)); - } else { - computation_map[computation_name] = - module->AddEmbeddedComputation(std::move(computation)); - } + // Don't uniquify names because we want names to be stable across + // serialization and deserialization. + computation_map[computation_name] = module->AddComputationInternal( + std::move(computation), + /*is_entry=*/proto.entry_computation_name() == computation_name, + /*uniquify_names=*/false); } TF_RET_CHECK(module->entry_computation_ != nullptr); + // Because we didn't uniquify the names, double-check that the instruction and + // computation names are unique from the proto. + tensorflow::gtl::FlatSet computation_names; + tensorflow::gtl::FlatSet instruction_names; + for (HloComputation* computation : module->computations()) { + if (computation->IsFusionComputation()) { + continue; + } + + TF_RET_CHECK(!ContainsKey(computation_names, computation->name())) + << "Computation name is not unique: " << computation->name(); + computation_names.insert(computation->name()); + for (HloInstruction* instruction : computation->instructions()) { + TF_RET_CHECK(!ContainsKey(instruction_names, instruction->name())) + << "Instruction name is not unique: " << instruction->name(); + instruction_names.insert(instruction->name()); + } + } + return std::move(module); } diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h index 96c17d6297..2ac4244e5c 100644 --- a/tensorflow/compiler/xla/service/hlo_module.h +++ b/tensorflow/compiler/xla/service/hlo_module.h @@ -182,7 +182,8 @@ class HloModule { private: HloComputation* AddComputationInternal( - std::unique_ptr computation); + std::unique_ptr computation, bool is_entry, + bool uniquify_names); const string name_; HloModuleConfig config_; -- GitLab From e43e514b7418756a828c6a0f60e43aa6a638e961 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Thu, 26 Oct 2017 04:43:45 -0700 Subject: [PATCH 401/573] TFE: Add incompatibility error and doc to add_check_numerics_ops() PiperOrigin-RevId: 173522273 --- tensorflow/contrib/eager/python/BUILD | 1 + tensorflow/contrib/eager/python/tfe_test.py | 8 ++++++++ tensorflow/python/BUILD | 1 + tensorflow/python/ops/numerics.py | 15 +++++++++++++++ 4 files changed, 25 insertions(+) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 340dca7e1a..adfaaa010a 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -18,6 +18,7 @@ py_library( ":saver", ":summary_writer", "//tensorflow/python:framework_ops", + "//tensorflow/python:numerics", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", "//tensorflow/python/eager:backprop", diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index eabff7f0a8..d8a38923a3 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -24,6 +24,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import numerics from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -101,6 +102,13 @@ class TFETest(test_util.TensorFlowTestCase): devices = tfe.list_devices() self.assertEqual(len(devices) - 1, tfe.num_gpus()) + def testAddCheckNumericsOpsRaisesError(self): + with self.assertRaisesRegexp( + RuntimeError, + r'add_check_numerics_ops\(\) is not compatible with eager execution'): + numerics.add_check_numerics_ops() + + if __name__ == '__main__': tfe.enable_eager_execution() test.main() diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 953aa566f0..e2be7e8e9a 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1921,6 +1921,7 @@ py_library( ":array_ops", ":control_flow_ops", ":framework_for_generated_wrappers", + "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py index 4e5d4bd9a1..f3558fda9c 100644 --- a/tensorflow/python/ops/numerics.py +++ b/tensorflow/python/ops/numerics.py @@ -19,6 +19,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -62,7 +63,21 @@ def add_check_numerics_ops(): Raises: ValueError: If the graph contains any numeric operations in a control flow structure. + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Not compatible with eager execution. To check for `Inf`s and `NaN`s under + eager execution, call tfe.seterr(inf_or_nan='raise') once before executing + the checked operations. + @enc_compatibility """ + if context.in_eager_mode(): + raise RuntimeError( + "add_check_numerics_ops() is not compatible with eager execution. " + "To check for Inf's and NaN's under eager execution, call " + "tfe.seterr(inf_or_nan='raise') once before executing the " + "checked operations.") + check_op = [] # This code relies on the ordering of ops in get_operations(). # The producer of a tensor always comes before that tensor's consumer in -- GitLab From 10a183353513d61863072cd47776bf9e488397a2 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 26 Oct 2017 09:06:31 -0400 Subject: [PATCH 402/573] Fix list formatting Markdown needs a blank line before the list to render it correctly. https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig#replace --- tensorflow/python/estimator/run_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py index 372f01dc82..d71964d2ec 100644 --- a/tensorflow/python/estimator/run_config.py +++ b/tensorflow/python/estimator/run_config.py @@ -528,6 +528,7 @@ class RunConfig(object): """Returns a new instance of `RunConfig` replacing specified properties. Only the properties in the following list are allowed to be replaced: + - `model_dir`. - `tf_random_seed`, - `save_summary_steps`, -- GitLab From b24b82ffe66e4ba4b520d5f3c8b5b7ad73f413b1 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 26 Oct 2017 08:01:09 -0700 Subject: [PATCH 403/573] [TF:XLA] Mark the "paddings" argument to PadV2 as a compile-time constant. PiperOrigin-RevId: 173538320 --- tensorflow/compiler/tests/binary_ops_test.py | 14 ++++++++++++++ tensorflow/compiler/tf2xla/const_analysis.cc | 1 + 2 files changed, 15 insertions(+) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index b387467246..9a225b32f8 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -707,6 +707,20 @@ class BinaryOpsTest(XLATestCase): [0, 0, 0, 0, 0, 0]], dtype=dtype)) + self._testBinary( + lambda x, y: array_ops.pad(x, y, constant_values=7), + np.array( + [[1, 2, 3], [4, 5, 6]], dtype=dtype), + np.array( + [[0, 3], [2, 1]], dtype=np.int32), + expected=np.array( + [[7, 7, 1, 2, 3, 7], + [7, 7, 4, 5, 6, 7], + [7, 7, 7, 7, 7, 7], + [7, 7, 7, 7, 7, 7], + [7, 7, 7, 7, 7, 7]], + dtype=dtype)) + def testMirrorPad(self): mirror_pad = lambda t, paddings: array_ops.pad(t, paddings, "REFLECT") for dtype in self.numeric_types: diff --git a/tensorflow/compiler/tf2xla/const_analysis.cc b/tensorflow/compiler/tf2xla/const_analysis.cc index bf75f85db0..102a2cf07b 100644 --- a/tensorflow/compiler/tf2xla/const_analysis.cc +++ b/tensorflow/compiler/tf2xla/const_analysis.cc @@ -67,6 +67,7 @@ Status BackwardsConstAnalysis(const Graph& g, {"Min", "reduction_indices"}, {"OneHot", "depth"}, {"Pad", "paddings"}, + {"PadV2", "paddings"}, {"MirrorPad", "paddings"}, {"Prod", "reduction_indices"}, {"RandomStandardNormal", "shape"}, -- GitLab From bfa539c03cd1555024fc04f4974e531c46b24e07 Mon Sep 17 00:00:00 2001 From: Malcolm Reynolds Date: Thu, 26 Oct 2017 08:42:37 -0700 Subject: [PATCH 404/573] Automated g4 rollback of changelist 173456597 PiperOrigin-RevId: 173542536 --- .../core/common_runtime/gpu/gpu_device.cc | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc index 2c906ed220..12d44cc6b7 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_device.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc @@ -652,34 +652,6 @@ Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options, if (static_cast(n) > valid_gpu_ids.size()) { n = valid_gpu_ids.size(); } - // Save the original device. - int original_device = 0; - cudaError_t err = cudaGetDevice(&original_device); - if (err != cudaSuccess) { - return errors::Internal("cudaGetDevice() failed. Status: ", - cudaGetErrorString(err)); - } - // Force to implicitly initialize CUDA runtime on each valid GPU before - // CreateGPUDevice(). - for (int gpu_id : valid_gpu_ids) { - err = cudaSetDevice(gpu_id); - if (err != cudaSuccess) { - return errors::Internal("cudaSetDevice() on GPU:", gpu_id, - " failed. Status: ", cudaGetErrorString(err)); - } - err = cudaFree(nullptr); - if (err != cudaSuccess) { - return errors::Internal( - "CUDA runtime implicit initialization on GPU:", gpu_id, - " failed. Status: ", cudaGetErrorString(err)); - } - } - // Reset to the original device. - err = cudaSetDevice(original_device); - if (err != cudaSuccess) { - return errors::Internal("cudaSetDevice() on GPU:", original_device, - " failed. Status: ", cudaGetErrorString(err)); - } for (int i = 0; i < n; i++) { BaseGPUDevice* gpu_device; TF_RETURN_IF_ERROR(CreateGPUDevice( -- GitLab From 8269c66ed239556d458c6bbd369ded206081ceb9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 10:13:19 -0700 Subject: [PATCH 405/573] K-FAC: register_additional_minibatch() for CategoricalLogitsNegativeLogProbLoss PiperOrigin-RevId: 173553770 --- .../contrib/kfac/python/kernel_tests/BUILD | 26 +-- .../kernel_tests/loss_functions_test.py | 57 +++++++ .../contrib/kfac/python/ops/loss_functions.py | 151 ++++++++++++++---- 3 files changed, 192 insertions(+), 42 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD index 8980f03092..5d86373a23 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD +++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD @@ -81,18 +81,6 @@ py_test( ], ) -py_test( - name = "loss_functions_test", - srcs = ["loss_functions_test.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/contrib/kfac/python/ops:loss_functions", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:constant_op", - ], -) - py_test( name = "optimizer_test", srcs = ["optimizer_test.py"], @@ -141,6 +129,20 @@ py_test( ], ) +py_test( + name = "loss_functions_test", + srcs = ["loss_functions_test.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/contrib/kfac/python/ops:loss_functions", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:framework_ops", + "//third_party/py/numpy", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py index 86dd839896..87339cb059 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py @@ -18,8 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import numpy as np + from tensorflow.contrib.kfac.python.ops import loss_functions from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -40,5 +43,59 @@ class InsertSliceInZerosTest(test.TestCase): self.assertAllEqual(expected_output_array, actual_output_array) +class CategoricalLogitsNegativeLogProbLossTest(test.TestCase): + + def testSample(self): + """Ensure samples can be drawn.""" + with ops.Graph().as_default(), self.test_session() as sess: + logits = np.asarray([ + [0., 0., 0.], # + [1., -1., 0.] + ]).astype(np.float32) + loss = loss_functions.CategoricalLogitsNegativeLogProbLoss( + array_ops.constant(logits)) + sample = loss.sample(42) + sample = sess.run(sample) + self.assertEqual(sample.shape, (2,)) + + def testEvaluateOnTargets(self): + """Ensure log probability can be evaluated correctly.""" + with ops.Graph().as_default(), self.test_session() as sess: + logits = np.asarray([ + [0., 0., 0.], # + [1., -1., 0.] + ]).astype(np.float32) + targets = np.asarray([2, 1]).astype(np.int32) + loss = loss_functions.CategoricalLogitsNegativeLogProbLoss( + array_ops.constant(logits), targets=array_ops.constant(targets)) + neg_log_prob = loss.evaluate() + neg_log_prob = sess.run(neg_log_prob) + + # Calculate explicit log probability of targets. + probs = np.exp(logits) / np.sum(np.exp(logits), axis=1, keepdims=True) + log_probs = np.log([ + probs[0, targets[0]], # + probs[1, targets[1]] + ]) + expected_log_prob = np.sum(log_probs) + + self.assertAllClose(neg_log_prob, -expected_log_prob) + + def testEvaluateOnSample(self): + """Ensure log probability of a sample can be drawn.""" + with ops.Graph().as_default(), self.test_session() as sess: + logits = np.asarray([ + [0., 0., 0.], # + [1., -1., 0.] + ]).astype(np.float32) + loss = loss_functions.CategoricalLogitsNegativeLogProbLoss( + array_ops.constant(logits)) + neg_log_prob = loss.evaluate_on_sample(42) + + # Simply ensure this doesn't crash. As the output is random, it's + # difficult to say if the output is correct or not... + neg_log_prob = sess.run(neg_log_prob) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py index 69d97f0b5b..3cfde7f9ab 100644 --- a/tensorflow/contrib/kfac/python/ops/loss_functions.py +++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py @@ -42,8 +42,14 @@ class LossFunction(object): use this class. It depends on the use case. """ - def __init__(self, targets=None): - self._targets = targets + @abc.abstractproperty + def targets(self): + """The targets being predicted by the model. + + Returns: + None or Tensor of appropriate shape for calling self._evaluate() on. + """ + pass @abc.abstractproperty def inputs(self): @@ -51,16 +57,25 @@ class LossFunction(object): pass def evaluate(self): - """Evaluate the loss function.""" - if self._targets is not None: + """Evaluate the loss function on the targets.""" + if self.targets is not None: # We treat the targets as "constant". It's only the inputs that get # "back-propped" through. - return self._evaluate(array_ops.stop_gradient(self._targets)) + return self._evaluate(array_ops.stop_gradient(self.targets)) else: raise Exception("Cannot evaluate losses with unspecified targets.") @abc.abstractmethod def _evaluate(self, targets): + """Evaluates the log probability of the targets. + + Args: + targets: Tensor that distribution can calculate log_prob() of. + + Returns: + log probability of each target, summed across all targets. + """ + pass @abc.abstractmethod @@ -166,9 +181,9 @@ class LossFunction(object): class NegativeLogProbLoss(LossFunction): """Abstract base class for loss functions that are negative log probs.""" - def __init__(self, targets=None, seed=None): + def __init__(self, seed=None): self._default_seed = seed - super(NegativeLogProbLoss, self).__init__(targets=targets) + super(NegativeLogProbLoss, self).__init__() @property def inputs(self): @@ -176,6 +191,7 @@ class NegativeLogProbLoss(LossFunction): @abc.abstractproperty def params(self): + """Parameters to the underlying distribution.""" pass @abc.abstractmethod @@ -281,9 +297,18 @@ class NegativeLogProbLoss(LossFunction): @abc.abstractmethod def sample(self, seed): + """Sample 'targets' from the underlying distribution.""" pass def evaluate_on_sample(self, seed=None): + """Evaluates the log probability on a random sample. + + Args: + seed: int or None. Random seed for this draw from the distribution. + + Returns: + Log probability of sampled targets, summed across examples. + """ if seed is None: seed = self._default_seed # We treat the targets as "constant". It's only the inputs that get @@ -328,16 +353,19 @@ class NaturalParamsNegativeLogProbLoss(NegativeLogProbLoss): class DistributionNegativeLogProbLoss(NegativeLogProbLoss): """Base class for neg log prob losses that use the TF Distribution classes.""" - def __init__(self, dist, targets=None, seed=None): - self._dist = dist - super(DistributionNegativeLogProbLoss, self).__init__( - targets=targets, seed=seed) + def __init__(self, seed=None): + super(DistributionNegativeLogProbLoss, self).__init__(seed=seed) + + @abc.abstractproperty + def dist(self): + """The underlying tf.distributions.Distribution.""" + pass def _evaluate(self, targets): - return -math_ops.reduce_sum(self._dist.log_prob(targets)) + return -math_ops.reduce_sum(self.dist.log_prob(targets)) def sample(self, seed): - return self._dist.sample(seed=seed) + return self.dist.sample(seed=seed) class NormalMeanNegativeLogProbLoss(DistributionNegativeLogProbLoss, @@ -355,11 +383,18 @@ class NormalMeanNegativeLogProbLoss(DistributionNegativeLogProbLoss, """ def __init__(self, mean, var=0.5, targets=None, seed=None): - dist = normal.Normal(loc=mean, scale=var**0.5) self._mean = mean self._var = var - super(NormalMeanNegativeLogProbLoss, self).__init__( - dist, targets=targets, seed=seed) + self._targets = targets + super(NormalMeanNegativeLogProbLoss, self).__init__(seed=seed) + + @property + def targets(self): + return self._targets + + @property + def dist(self): + return normal.Normal(loc=self._mean, scale=math_ops.sqrt(self._var)) @property def params(self): @@ -416,10 +451,16 @@ class NormalMeanVarianceNegativeLogProbLoss(DistributionNegativeLogProbLoss): self._mean = mean self._variance = variance self._scale = math_ops.sqrt(variance) - dist = normal.Normal(loc=self._mean, scale=self._scale) - super(NormalMeanVarianceNegativeLogProbLoss, self).__init__(dist, - targets=targets, - seed=seed) + self._targets = targets + super(NormalMeanVarianceNegativeLogProbLoss, self).__init__(seed=seed) + + @property + def targets(self): + return self._targets + + @property + def dist(self): + return normal.Normal(loc=self._mean, scale=self._scale) @property def params(self): @@ -534,12 +575,53 @@ class CategoricalLogitsNegativeLogProbLoss(DistributionNegativeLogProbLoss, """ def __init__(self, logits, targets=None, seed=None): - dist = categorical.Categorical(logits=logits) - self._logits = logits - self._probs = dist.probs - self._sqrt_probs = math_ops.sqrt(self._probs) - super(CategoricalLogitsNegativeLogProbLoss, self).__init__( - dist, targets=targets, seed=seed) + """Instantiates a CategoricalLogitsNegativeLogProbLoss. + + Args: + logits: Tensor of shape [batch_size, output_size]. Parameters for + underlying distribution. + targets: None or Tensor of shape [output_size]. Each elements contains an + index in [0, output_size). + seed: int or None. Default random seed when sampling. + """ + self._logits_components = [] + self._targets_components = [] + self.register_additional_minibatch(logits, targets=targets) + super(CategoricalLogitsNegativeLogProbLoss, self).__init__(seed=seed) + + def register_additional_minibatch(self, logits, targets=None): + """Register an additiona minibatch's worth of parameters. + + Args: + logits: Tensor of shape [batch_size, output_size]. Parameters for + underlying distribution. + targets: None or Tensor of shape [batch_size, output_size]. Each row must + be a one-hot vector. + """ + self._logits_components.append(logits) + self._targets_components.append(targets) + + @property + def _logits(self): + return array_ops.concat(self._logits_components, axis=0) + + @property + def targets(self): + if all(target is None for target in self._targets_components): + return None + return array_ops.concat(self._targets_components, axis=0) + + @property + def dist(self): + return categorical.Categorical(logits=self._logits) + + @property + def _probs(self): + return self.dist.probs + + @property + def _sqrt_probs(self): + return math_ops.sqrt(self._probs) @property def params(self): @@ -595,12 +677,21 @@ class MultiBernoulliNegativeLogProbLoss(DistributionNegativeLogProbLoss, """ def __init__(self, logits, targets=None, seed=None): - dist = bernoulli.Bernoulli(logits=logits) self._logits = logits - self._probs = dist.probs + self._targets = targets + super(MultiBernoulliNegativeLogProbLoss, self).__init__(seed=seed) + + @property + def targets(self): + return self._targets - super(MultiBernoulliNegativeLogProbLoss, self).__init__( - dist, targets=targets, seed=seed) + @property + def dist(self): + return bernoulli.Bernoulli(logits=self._logits) + + @property + def _probs(self): + return self.dist.probs @property def params(self): -- GitLab From f0aa811ee71ef2a54c67a1311557b331379a56e6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 10:48:45 -0700 Subject: [PATCH 406/573] Use the new robust implementation of log determinant (linalg.slogdet) in LinearOperator. Move creation of aliases into linalg_impl, such that these can be used inside TensorFlow without creating circular dependencies. PiperOrigin-RevId: 173558892 --- tensorflow/python/ops/linalg/BUILD | 5 +-- tensorflow/python/ops/linalg/linalg.py | 40 +++---------------- tensorflow/python/ops/linalg/linalg_impl.py | 28 +++++++++++++ .../python/ops/linalg/linear_operator.py | 4 +- .../ops/linalg/linear_operator_test_util.py | 3 +- 5 files changed, 38 insertions(+), 42 deletions(-) diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD index b88e72a6f3..ce8c1580fe 100644 --- a/tensorflow/python/ops/linalg/BUILD +++ b/tensorflow/python/ops/linalg/BUILD @@ -9,15 +9,13 @@ py_library( srcs = glob(["*.py"]), srcs_version = "PY2AND3", deps = [ - "//tensorflow/python:array_ops", + ":linalg_impl", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", "//tensorflow/python:random_ops", - "//tensorflow/python:special_math_ops", "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//third_party/py/numpy", @@ -33,6 +31,7 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:special_math_ops", ], ) diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py index 02ceb65e2a..5369007a56 100644 --- a/tensorflow/python/ops/linalg/linalg.py +++ b/tensorflow/python/ops/linalg/linalg.py @@ -18,12 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_linalg_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import special_math_ops - # go/tf-wildcard-import # pylint: disable=wildcard-import,unused-import from tensorflow.python.ops.linalg.linalg_impl import * @@ -36,39 +30,15 @@ from tensorflow.python.ops.linalg.linear_operator_low_rank_update import * from tensorflow.python.ops.linalg.linear_operator_lower_triangular import * # pylint: enable=wildcard-import -# Linear algebra ops. -band_part = array_ops.matrix_band_part -cholesky = linalg_ops.cholesky -cholesky_solve = linalg_ops.cholesky_solve -det = linalg_ops.matrix_determinant -# pylint: disable=protected-access -slogdet = gen_linalg_ops._log_matrix_determinant -# pylint: disable=protected-access -diag = array_ops.matrix_diag -diag_part = array_ops.matrix_diag_part -eigh = linalg_ops.self_adjoint_eig -eigvalsh = linalg_ops.self_adjoint_eigvals -einsum = special_math_ops.einsum -eye = linalg_ops.eye -inv = linalg_ops.matrix_inverse -lstsq = linalg_ops.matrix_solve_ls -norm = linalg_ops.norm -qr = linalg_ops.qr -set_diag = array_ops.matrix_set_diag -solve = linalg_ops.matrix_solve -svd = linalg_ops.svd -tensordot = math_ops.tensordot -trace = math_ops.trace -transpose = array_ops.matrix_transpose -triangular_solve = linalg_ops.matrix_triangular_solve - # Seal API. +# pylint: disable=undefined-variable del absolute_import -del array_ops del division +del print_function +del ops +del array_ops del gen_linalg_ops del linalg_ops del math_ops -del ops -del print_function del special_math_ops +# pylint: enable=undefined-variable diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py index 1fdec2b51b..04a15e3e5b 100644 --- a/tensorflow/python/ops/linalg/linalg_impl.py +++ b/tensorflow/python/ops/linalg/linalg_impl.py @@ -21,7 +21,35 @@ from __future__ import print_function from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_linalg_ops +from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import special_math_ops + +# Linear algebra ops. +band_part = array_ops.matrix_band_part +cholesky = linalg_ops.cholesky +cholesky_solve = linalg_ops.cholesky_solve +det = linalg_ops.matrix_determinant +# pylint: disable=protected-access +slogdet = gen_linalg_ops._log_matrix_determinant +# pylint: disable=protected-access +diag = array_ops.matrix_diag +diag_part = array_ops.matrix_diag_part +eigh = linalg_ops.self_adjoint_eig +eigvalsh = linalg_ops.self_adjoint_eigvals +einsum = special_math_ops.einsum +eye = linalg_ops.eye +inv = linalg_ops.matrix_inverse +lstsq = linalg_ops.matrix_solve_ls +norm = linalg_ops.norm +qr = linalg_ops.qr +set_diag = array_ops.matrix_set_diag +solve = linalg_ops.matrix_solve +svd = linalg_ops.svd +tensordot = math_ops.tensordot +trace = math_ops.trace +transpose = array_ops.matrix_transpose +triangular_solve = linalg_ops.matrix_triangular_solve def logdet(matrix, name=None): diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py index 0d04e29eb3..27e0f17020 100644 --- a/tensorflow/python/ops/linalg/linear_operator.py +++ b/tensorflow/python/ops/linalg/linear_operator.py @@ -693,8 +693,8 @@ class LinearOperator(object): if self._can_use_cholesky(): diag = array_ops.matrix_diag_part(self._get_cached_chol()) return 2 * math_ops.reduce_sum(math_ops.log(diag), reduction_indices=[-1]) - abs_det = math_ops.abs(self.determinant()) - return math_ops.log(abs_det) + _, log_abs_det = linalg.slogdet(self._matrix) + return log_abs_det def log_abs_determinant(self, name="log_abs_det"): """Log absolute value of determinant for every batch member. diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py index 4a601047b6..3d0ea3e11b 100644 --- a/tensorflow/python/ops/linalg/linear_operator_test_util.py +++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py @@ -191,8 +191,7 @@ class LinearOperatorDerivedClassTest(test.TestCase): operator, mat, feed_dict = self._operator_and_mat_and_feed_dict( shape, dtype, use_placeholder=use_placeholder) op_log_abs_det = operator.log_abs_determinant() - mat_log_abs_det = math_ops.log( - math_ops.abs(linalg_ops.matrix_determinant(mat))) + _, mat_log_abs_det = linalg.slogdet(mat) if not use_placeholder: self.assertAllEqual(shape[:-2], op_log_abs_det.get_shape()) op_log_abs_det_v, mat_log_abs_det_v = sess.run( -- GitLab From 0bbee64143ca76d83ff284fb80a9b59c367bac28 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Thu, 26 Oct 2017 10:58:51 -0700 Subject: [PATCH 407/573] Automated g4 rollback of changelist 173494053 PiperOrigin-RevId: 173560463 --- tensorflow/contrib/cudnn_rnn/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/cudnn_rnn/__init__.py b/tensorflow/contrib/cudnn_rnn/__init__.py index bc44562b50..87ba834770 100644 --- a/tensorflow/contrib/cudnn_rnn/__init__.py +++ b/tensorflow/contrib/cudnn_rnn/__init__.py @@ -30,14 +30,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnGRU -from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnLSTM -from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNRelu -from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNTanh from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleGRUCell from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleLSTMCell +from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRU from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRUSaveable +from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTM from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTMSaveable +from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNRelu from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNReluSaveable from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNTanhSaveable -- GitLab From 50914f04dc5eb28695933bf94a8e8fe221371b7c Mon Sep 17 00:00:00 2001 From: ted chang Date: Wed, 25 Oct 2017 12:41:57 -0700 Subject: [PATCH 408/573] Changed GPU driver version assumption Fixes #9669 --- tensorflow/stream_executor/cuda/cuda_diagnostics.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc index bf81b9c0ad..00506fa54b 100644 --- a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc +++ b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc @@ -76,10 +76,10 @@ string DriverVersionStatusToString(port::StatusOr version) { port::StatusOr StringToDriverVersion(const string &value) { std::vector pieces = port::Split(value, '.'); - if (pieces.size() != 2 && pieces.size() != 3) { + if (pieces.size() < 2 || pieces.size() > 4) { return port::Status{ port::error::INVALID_ARGUMENT, - port::Printf("expected %%d.%%d or %%d.%%d.%%d form for driver version; got \"%s\"", + port::Printf("expected %%d.%%d, %%d.%%d.%%d, or %%d.%%d.%%d.%%d form for driver version; got \"%s\"", value.c_str())}; } -- GitLab From fcf3e00d04bc911e2af9f2cbc13edee30dc03f7c Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Thu, 26 Oct 2017 11:49:59 -0700 Subject: [PATCH 409/573] Implement __int__, __float__ for EagerTensors PiperOrigin-RevId: 173569138 --- tensorflow/python/eager/ops_test.py | 14 ++++++++++++++ tensorflow/python/framework/ops.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index e34587d5b1..e86073d6b2 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -331,6 +331,20 @@ class OpsTest(test_util.TensorFlowTestCase): x.set_shape(tensor_shape.TensorShape([None, 2])) self.assertEqual(x.get_shape(), (1, 2)) + def testCastScalarToPrimitiveTypes(self): + x = constant_op.constant(1.3) + self.assertIsInstance(int(x), int) + self.assertEqual(int(x), 1) + self.assertIsInstance(float(x), float) + self.assertAllClose(float(x), 1.3) + + def testCastNonScalarToPrimitiveTypesFails(self): + x = constant_op.constant([1.3, 2]) + with self.assertRaises(TypeError): + int(x) + with self.assertRaises(TypeError): + float(x) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 61a5a4fcae..eceacb42d9 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -623,6 +623,14 @@ class _EagerTensorBase(Tensor): raise ValueError("Resource handles are not convertible to numpy.") return self.cpu()._numpy() # pylint: disable=protected-access + # __int__ and __float__ may copy the tensor to CPU and + # only work for scalars; values are cast as per numpy. + def __int__(self): + return int(self.numpy()) + + def __float__(self): + return float(self.numpy()) + def __array__(self): return np.array(self.numpy()) -- GitLab From a80f91bd8a9733800275b0b1328747770c7e46e8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 12:31:07 -0700 Subject: [PATCH 410/573] K-FAC: Support for multiple minibatches with register_categorical_predictive_distribution(). PiperOrigin-RevId: 173574699 --- .../kernel_tests/layer_collection_test.py | 53 +++++++++++++++++-- .../kfac/python/ops/layer_collection.py | 46 +++++++++++++--- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 432937d803..4f27ceced9 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -214,8 +214,8 @@ class LayerCollectionTest(test.TestCase): self.assertEqual(1, len(lc.losses)) # Add logits to same loss function. - with self.assertRaises(NotImplementedError): - lc.register_categorical_predictive_distribution(logits, name='loss1') + lc.register_categorical_predictive_distribution( + logits, name='loss1', reuse=True) self.assertEqual(1, len(lc.losses)) # Add another new loss function. @@ -228,11 +228,58 @@ class LayerCollectionTest(test.TestCase): logits = linalg_ops.eye(2) lc = layer_collection.LayerCollection() - # Create a new loss function by name. + # Create a new loss function with default names. lc.register_categorical_predictive_distribution(logits) lc.register_categorical_predictive_distribution(logits) self.assertEqual(2, len(lc.losses)) + def testCategoricalPredictiveDistributionMultipleMinibatches(self): + """Ensure multiple minibatches are registered.""" + with ops.Graph().as_default(): + batch_size = 3 + output_size = 2 + logits = array_ops.zeros([batch_size, output_size]) + targets = array_ops.ones([batch_size], dtype=dtypes.int32) + lc = layer_collection.LayerCollection() + + # Create a new loss function. + lc.register_categorical_predictive_distribution( + logits, targets=targets, name='loss1') + + # Can add when reuse=True + lc.register_categorical_predictive_distribution( + logits, targets=targets, name='loss1', reuse=True) + + # Can add when reuse=VARIABLE_SCOPE and reuse=True there. + with variable_scope.variable_scope( + variable_scope.get_variable_scope(), reuse=True): + lc.register_categorical_predictive_distribution( + logits, + targets=targets, + name='loss1', + reuse=layer_collection.VARIABLE_SCOPE) + + # Can't add when reuse=False + with self.assertRaises(KeyError): + lc.register_categorical_predictive_distribution( + logits, targets=targets, name='loss1', reuse=False) + + # Can't add when reuse=VARIABLE_SCOPE and reuse=False there. + with self.assertRaises(KeyError): + lc.register_categorical_predictive_distribution( + logits, + targets=targets, + name='loss1', + reuse=layer_collection.VARIABLE_SCOPE) + + self.assertEqual(len(lc.losses), 1) + loss = lc.losses[0] + + # Three successful registrations. + self.assertEqual(loss.params.shape.as_list(), + [3 * batch_size, output_size]) + self.assertEqual(loss.targets.shape.as_list(), [3 * batch_size]) + def testRegisterCategoricalPredictiveDistributionBatchSize1(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index cd711d0561..2b9958a46a 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -342,7 +342,8 @@ class LayerCollection(object): logits, seed=None, targets=None, - name=None): + name=None, + reuse=VARIABLE_SCOPE): """Registers a categorical predictive distribution. Args: @@ -355,15 +356,46 @@ class LayerCollection(object): (Default: None) name: (OPTIONAL) str or None. Unique name for this loss function. If None, a new name is generated. (Default: None) + reuse: (OPTIONAL) bool or str. If True, reuse an existing FisherBlock. + If False, create a new FisherBlock. If VARIABLE_SCOPE, use + tf.get_variable_scope().reuse. + + Raises: + ValueError: If reuse=True and name != None. + ValueError: If reuse=True and seed != None. + KeyError: If reuse=True and no existing LossFunction with 'name' found. + KeyError: If reuse=False and existing LossFunction with 'name' found. """ name = name or self._graph.unique_name( "register_categorical_predictive_distribution") - if name in self._loss_dict: - raise NotImplementedError( - "Adding logits to an existing LossFunction not yet supported.") - loss = lf.CategoricalLogitsNegativeLogProbLoss( - logits, targets=targets, seed=seed) - self._loss_dict[name] = loss + + if reuse == VARIABLE_SCOPE: + reuse = variable_scope.get_variable_scope().reuse + + if reuse: + if name is None: + raise ValueError( + "If reuse is enabled, loss function's name must be set.") + if seed is not None: + raise ValueError( + "Seed can only be specified at LossFunction instantiation.") + + loss = self._loss_dict.get(name, None) + + if loss is None: + raise KeyError( + "Unable to find loss function named {}. Create a new LossFunction " + "with reuse=False.".format(name)) + + loss.register_additional_minibatch(logits, targets=targets) + else: + if name in self._loss_dict: + raise KeyError( + "Loss function named {} already exists. Set reuse=True to append " + "another minibatch.".format(name)) + loss = lf.CategoricalLogitsNegativeLogProbLoss( + logits, targets=targets, seed=seed) + self._loss_dict[name] = loss def register_normal_predictive_distribution(self, mean, -- GitLab From bab6e69913f7fd0ad59a93e092ac28720b99a05c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 12:43:08 -0700 Subject: [PATCH 411/573] Updating documentation of _remove_squeezable_dimensions (in python/ops/metrics_impl.py) and removing duplicate function in contrib/metrics/python/ops/metric_ops.py. PiperOrigin-RevId: 173576149 --- .../contrib/metrics/python/ops/metric_ops.py | 68 ++++--------------- .../metrics/python/ops/metric_ops_test.py | 2 +- tensorflow/python/ops/metrics_impl.py | 18 +++-- 3 files changed, 24 insertions(+), 64 deletions(-) diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 675c49dfc3..50b9c4afde 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -28,7 +28,6 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops -from tensorflow.python.ops import confusion_matrix from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics @@ -223,7 +222,7 @@ def streaming_true_negatives(predictions, with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) @@ -654,7 +653,7 @@ def _true_negatives(labels, with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) @@ -715,7 +714,7 @@ def streaming_false_positive_rate(predictions, """ with variable_scope.variable_scope(name, 'false_positive_rate', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) @@ -803,7 +802,7 @@ def streaming_false_negative_rate(predictions, """ with variable_scope.variable_scope(name, 'false_negative_rate', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) @@ -896,7 +895,7 @@ def _streaming_confusion_matrix_at_thresholds(predictions, if include not in all_includes: raise ValueError('Invaild key: %s.' % include) - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -1284,8 +1283,10 @@ def streaming_precision_recall_at_equal_thresholds(predictions, math_ops.cast(1.0, dtype=predictions.dtype), message='predictions must be in [0, 1]') ]): - predictions, labels, weights = _remove_squeezable_dimensions( - predictions=predictions, labels=labels, weights=weights) + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access + predictions=predictions, + labels=labels, + weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -2597,7 +2598,7 @@ def streaming_covariance(predictions, """ with variable_scope.variable_scope(name, 'covariance', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) count = _create_local('count', []) @@ -2731,7 +2732,7 @@ def streaming_pearson_correlation(predictions, """ with variable_scope.variable_scope(name, 'pearson_r', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) # Broadcast weights here to avoid duplicate broadcasting in each call to @@ -2813,7 +2814,7 @@ def streaming_mean_cosine_distance(predictions, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) radial_diffs = math_ops.multiply(predictions, labels) @@ -3123,51 +3124,6 @@ def aggregate_metric_map(names_to_tuples): return dict(zip(metric_names, value_ops)), dict(zip(metric_names, update_ops)) -def _remove_squeezable_dimensions(predictions, labels, weights): - """Squeeze last dim if needed. - - Squeezes `predictions` and `labels` if their rank differs by 1. - Squeezes `weights` if its rank is 1 more than the new rank of `predictions` - - This will use static shape if available. Otherwise, it will add graph - operations, which could result in a performance hit. - - Args: - predictions: Predicted values, a `Tensor` of arbitrary dimensions. - labels: Label values, a `Tensor` whose dimensions match `predictions`. - weights: Optional weight `Tensor`. It will be squeezed if its rank is 1 - more than the new rank of `predictions` - - Returns: - Tuple of `predictions`, `labels` and `weights`, possibly with the last - dimension squeezed. - """ - labels, predictions = confusion_matrix.remove_squeezable_dimensions( - labels, predictions) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - - if weights is not None: - weights = ops.convert_to_tensor(weights) - predictions_shape = predictions.get_shape() - predictions_rank = predictions_shape.ndims - weights_shape = weights.get_shape() - weights_rank = weights_shape.ndims - - if (predictions_rank is not None) and (weights_rank is not None): - # Use static rank. - if weights_rank - predictions_rank == 1: - weights = array_ops.squeeze(weights, [-1]) - elif (weights_rank is - None) or (weights_shape.dims[-1].is_compatible_with(1)): - # Use dynamic rank - weights = control_flow_ops.cond( - math_ops.equal( - array_ops.rank(weights), - math_ops.add(array_ops.rank(predictions), 1)), - lambda: array_ops.squeeze(weights, [-1]), lambda: weights) - return predictions, labels, weights - - __all__ = [ 'aggregate_metric_map', 'aggregate_metrics', diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py index 6e038481e3..24d82a7eee 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py @@ -2131,7 +2131,7 @@ class StreamingPrecisionRecallAtEqualThresholdsTest(test.TestCase): 'recall': [1.0, 1.0, 0.0], 'thresholds': [0.0, 0.5, 1.0], }, - weights=[0.0, 0.5, 2.0, 0.0, 0.5, 1.0]) + weights=[[0.0, 0.5, 2.0, 0.0, 0.5, 1.0]]) class StreamingSpecificityAtSensitivityTest(test.TestCase): diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py index 1858834f97..68ec3c0101 100644 --- a/tensorflow/python/ops/metrics_impl.py +++ b/tensorflow/python/ops/metrics_impl.py @@ -52,10 +52,14 @@ def _local_variable(initial_value, validate_shape=True, name=None): def _remove_squeezable_dimensions(predictions, labels, weights): - """Internal version of `remove_squeezable_dimensions` which handles weights. + """Squeeze or expand last dim if needed. - Squeezes `predictions` and `labels` if their rank differs by 1. - Squeezes `weights` if its rank is 1 more than the new rank of `predictions` + Squeezes last dim of `predictions` or `labels` if their rank differs by 1 + (using confusion_matrix.remove_squeezable_dimensions). + Squeezes or expands last dim of `weights` if its rank differs by 1 from the + new rank of `predictions`. + + If `weights` is scalar, it is kept scalar. This will use static shape if available. Otherwise, it will add graph operations, which could result in a performance hit. @@ -63,12 +67,12 @@ def _remove_squeezable_dimensions(predictions, labels, weights): Args: predictions: Predicted values, a `Tensor` of arbitrary dimensions. labels: Optional label `Tensor` whose dimensions match `predictions`. - weights: Optional weight `Tensor`. It will be squeezed if its rank is 1 - more than the new rank of `predictions` + weights: Optional weight scalar or `Tensor` whose dimensions match + `predictions`. Returns: - Tuple of `predictions`, `labels` and `weights`, possibly with the last - dimension squeezed. + Tuple of `predictions`, `labels` and `weights`. Each of them possibly has + the last dimension squeezed, `weights` could be extended by one dimension. """ predictions = ops.convert_to_tensor(predictions) if labels is not None: -- GitLab From 3fae77a3d8f4c921d6e45b9a87a5637fd53b3071 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Thu, 26 Oct 2017 12:51:10 -0700 Subject: [PATCH 412/573] [TFXLA] Add grad_state for TPUReplicateContext PiperOrigin-RevId: 173577089 --- tensorflow/contrib/tpu/python/tpu/tpu.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py index fa5760953d..338a4304f3 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu.py @@ -146,6 +146,14 @@ class TPUReplicateContext(control_flow_ops.ControlFlowContext): if self._outer_context: self._outer_context.AddInnerOp(op) + @property + def grad_state(self): + # Define the gradient loop state associated with the TPUReplicateContext to + # be None as the TPUReplicateContext does not get nested nor does the + # grad_state outside the TPUReplicateContext affect the graph inside so the + # grad_state should be as if this is the top-level gradient state. + return None + def replicate(computation, inputs=None, -- GitLab From 3db4df07101b013eacc37de0e2ff990bdadb3219 Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Thu, 26 Oct 2017 13:26:50 -0700 Subject: [PATCH 413/573] fixed eval num_epochs. PiperOrigin-RevId: 173581816 --- tensorflow/docs_src/get_started/get_started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/get_started/get_started.md b/tensorflow/docs_src/get_started/get_started.md index 67fddfe809..8409962744 100644 --- a/tensorflow/docs_src/get_started/get_started.md +++ b/tensorflow/docs_src/get_started/get_started.md @@ -453,7 +453,7 @@ input_fn = tf.estimator.inputs.numpy_input_fn( train_input_fn = tf.estimator.inputs.numpy_input_fn( {"x": x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False) eval_input_fn = tf.estimator.inputs.numpy_input_fn( - {"x": x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False) + {"x": x_eval}, y_eval, batch_size=4, num_epochs=1, shuffle=False) # train estimator.train(input_fn=input_fn, steps=1000) -- GitLab From 24105d9a83dff9b46326373a7c4fd7fd254f32f0 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 26 Oct 2017 13:42:19 -0700 Subject: [PATCH 414/573] [XLA] Merge large parameter-shaped tuples into their users in DOT graphs. It's common to have a while loop whose body computation has one parameter, a giant tuple. Then we have to draw edges from that tuple to a bunch of get-tuple-element nodes, which are used throughout the while loop's body. This results in many long, difficult-to-follow edges. In practice, the big tuple really functions as N separate parameters. This patch represents it this way visually, erasing the big tuple and replacing it with the get-tuple-element users, which we style like parameters. Future work is figuring out how to do something similar for the tuple op at the bottom of while loop bodies. This will be harder, because it will require breaking the invariant that every HLO corresponds to zero or one nodes in the dot graph. PiperOrigin-RevId: 173584100 --- .../compiler/xla/service/hlo_graph_dumper.cc | 82 +++++++++++++++++-- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index b11b129c14..7b9cbeb6f4 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -342,6 +342,11 @@ class HloDotDumper { bool ShouldShowSubcomputation(const HloComputation* subcomp); bool ShouldShowFusionSubcomputation(const HloInstruction* instr); + + // We omit some nodes from the graph, instead drawing them inlined into the + // nodes that use them. + bool ShouldMergeIntoUsers(const HloInstruction* instr) const; + string DumpSubcomputation(const HloComputation* subcomp, const HloInstruction* parent_instr); string DumpComputation(const HloComputation* comp); @@ -352,7 +357,7 @@ class HloDotDumper { string GetInstructionNodeLabel(const HloInstruction* instr); string GetInstructionNodeMetadata(const HloInstruction* instr); string GetInstructionNodeExtraInfo(const HloInstruction* instr); - string GetInstructionNodeInlinedConstants(const HloInstruction* instr); + string GetInstructionNodeInlinedOperands(const HloInstruction* instr); void AddInstructionIncomingEdges(const HloInstruction* instr); // If instr has just one computation and it's trivial (e.g. "return param0 + @@ -668,12 +673,42 @@ string HloDotDumper::DumpRootTag() { to_id, node_body, node_shape, NodeColorAttributes(color)); } +bool HloDotDumper::ShouldMergeIntoUsers(const HloInstruction* instr) const { + // If a node: + // + // - is a tuple-shaped parameter, + // - is not a parameter to a fusion node, + // - has at least kMinUsersToOmit users shown, and + // - all of the shown users are get-tuple-elements, + // + // then we omit it from the graph, merging it with its users. + // + // This helps us handle the common case where a while loop body has one big + // tuple-shaped parameter. + const int kMinUsersToOmit = 3; + return instr->opcode() == HloOpcode::kParameter && + ShapeUtil::IsTuple(instr->shape()) && !instr->IsFused() && + std::count_if(instr->users().begin(), instr->users().end(), + [&](const HloInstruction* user) { + return filter_.Show(user); + }) > kMinUsersToOmit && + std::all_of(instr->users().begin(), instr->users().end(), + [&](const HloInstruction* user) { + return !filter_.Show(user) || + user->opcode() == HloOpcode::kGetTupleElement; + }); +} + string HloDotDumper::DumpInstruction(const HloInstruction* instr) { // We don't display constants as separate nodes; they're merged into their // users. if (instr->opcode() == HloOpcode::kConstant) { return ""; } + // Skip this node if it's merged into its users. + if (ShouldMergeIntoUsers(instr)) { + return ""; + } // Omit the fusion node if its subcomputation is drawn, since the // subcomputation will be drawn inline. if (instr->opcode() == HloOpcode::kFusion && @@ -689,7 +724,7 @@ string HloDotDumper::DumpInstruction(const HloInstruction* instr) { string node_label = GetInstructionNodeLabel(instr); string node_metadata = GetInstructionNodeMetadata(instr); string extra_info = GetInstructionNodeExtraInfo(instr); - string inlined_constants = GetInstructionNodeInlinedConstants(instr); + string inlined_constants = GetInstructionNodeInlinedOperands(instr); string trivial_subcomputation = GetInstructionTrivialComputationStr(instr); AddInstructionIncomingEdges(instr); @@ -717,7 +752,7 @@ string HloDotDumper::DumpInstruction(const HloInstruction* instr) { NodeColorAttributes(color)); } -string HloDotDumper::GetInstructionNodeInlinedConstants( +string HloDotDumper::GetInstructionNodeInlinedOperands( const HloInstruction* instr) { auto stringify_constant = [](const HloInstruction* constant) { if (ShapeUtil::IsEffectiveScalar(constant->shape())) { @@ -746,16 +781,44 @@ string HloDotDumper::GetInstructionNodeInlinedConstants( std::vector lines; for (int64 i = 0; i < instr->operand_count(); ++i) { const HloInstruction* operand = instr->operand(i); - if (operand->opcode() != HloOpcode::kConstant) { - continue; + optional operand_str; + if (operand->opcode() == HloOpcode::kConstant) { + operand_str = stringify_constant(operand); + } else if (ShouldMergeIntoUsers(operand)) { + // Special case: If the operand is a parameter, use its parameter number + // rather than its name, because that's generally how people think of the + // node. + if (operand->opcode() == HloOpcode::kParameter) { + operand_str = Printf("Parameter %lld", operand->parameter_number()); + } else { + operand_str = operand->name(); + } + } + + if (operand_str) { + if (instr->operand_count() > 1) { + lines.push_back(Printf("operand %lld = %s", i, *operand_str)); + } else { + lines.push_back(Printf("operand = %s", *operand_str)); + } } - lines.push_back( - Printf("operand %lld = %s", i, stringify_constant(operand))); } return Join(lines, "
"); } ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { + const auto kParameterColor = kOrange; + + // Special case: If this instruction has a parameter merged into it, paint it + // the same color as a parameter. + if (std::any_of(instr->operands().begin(), instr->operands().end(), + [&](const HloInstruction* operand) { + return operand->opcode() == HloOpcode::kParameter && + ShouldMergeIntoUsers(operand); + })) { + return kParameterColor; + } + // Pick different colors or shapes for instructions which are particularly // expensive (eg, dot) and those which are unusual in some way or unique // (eg, parameter). @@ -845,7 +908,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kReducePrecision: return kRed; case HloOpcode::kParameter: - return kOrange; + return kParameterColor; case HloOpcode::kBatchNormTraining: case HloOpcode::kBatchNormInference: case HloOpcode::kBatchNormGrad: @@ -1016,7 +1079,8 @@ void HloDotDumper::AddInstructionIncomingEdges(const HloInstruction* instr) { ShouldShowFusionSubcomputation(from)) { from = from->fused_expression_root(); } - if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant) { + if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant || + ShouldMergeIntoUsers(from)) { return; } VLOG(2) << "Adding edge from " << from->name() << " to " << to->name() -- GitLab From efe4c98b57baf2697c0c226157fe1efe8810f605 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 13:54:59 -0700 Subject: [PATCH 415/573] Add tests for OIHW filter format. PiperOrigin-RevId: 173586221 --- .../fused_conv2d_bias_activation_op_test.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py index 3b8f7d6ed7..2a18f3eeec 100644 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py +++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py @@ -159,9 +159,12 @@ class FusedConv2DBiasActivationTest(test.TestCase): def _DtypesToTest(self, use_gpu): return [dtypes.float32] + def _FilterFormatsToTest(self, use_gpu): + return ["HWIO", "OIHW"] + def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, bias, strides, padding, activation_mode, data_format, - dtype): + filter_format, dtype): """Verifies the output values of the convolution function. Args: @@ -174,6 +177,7 @@ class FusedConv2DBiasActivationTest(test.TestCase): padding: Padding type. activation_mode: Activation mode. data_format: Format of the data tensors. + filter_format: Filter format to use for the fused convolution. dtype: Data type for inputs and outputs. Returns: Symbolic tensor value and reference value that can be used to @@ -192,6 +196,9 @@ class FusedConv2DBiasActivationTest(test.TestCase): with self.test_session(use_gpu=True): t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype) t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype) + fused_t2 = t2 + if filter_format == "OIHW": + fused_t2 = HwioToOihw(t2) t3 = constant_op.constant(x3, shape=[bias_size], dtype=dtype) strides = [1] + strides + [1] if data_format == "NCHW": @@ -199,11 +206,12 @@ class FusedConv2DBiasActivationTest(test.TestCase): strides = test_util.NHWCToNCHW(strides) output = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( t1, - t2, + fused_t2, t3, strides=strides, padding=padding, data_format=data_format, + filter_format=filter_format, activation_mode=activation_mode) ref_conv_output = nn_ops.conv2d( t1, t2, strides=strides, padding=padding, data_format=data_format) @@ -268,9 +276,10 @@ class FusedConv2DBiasActivationTest(test.TestCase): ref_tensors = [] for (data_format, use_gpu) in GetTestConfigs(): for dtype in self._DtypesToTest(use_gpu): - result, expected = self._SetupValuesForDevice( - tensor_in_sizes, filter_in_sizes, bias, strides, padding, "Relu", - data_format, dtype) + for filter_format in self._FilterFormatsToTest(use_gpu): + result, expected = self._SetupValuesForDevice( + tensor_in_sizes, filter_in_sizes, bias, strides, padding, "Relu", + data_format, filter_format, dtype) tensors.append(result) ref_tensors.append(expected) with self.test_session() as sess: @@ -607,6 +616,10 @@ def NchwToNchwVectC(in_tensor): return array_ops.transpose(t, [0, 1, 3, 4, 2]) +def HwioToOihw(in_tensor): + return array_ops.transpose(in_tensor, [3, 2, 0, 1]) + + def SimulateFusedConv2dBiasActivationInt8(conv_input_scale, conv_input, kernel, padding, strides, side_input_scale, side_input, biases): -- GitLab From 00adbdb9bc4650e3fa594cfdd408b30cf38243bf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 13:55:54 -0700 Subject: [PATCH 416/573] Object-orient Metrics fixes: * Add examples for usage in both eager and graph execution. * Remove references to Evaluator in the doc string, to support stand-alone uses. * Remove the need to override reset() if you use non-zero initialization of some variable, by recording the initial values of variables. * Merge reset() into init_variables(). PiperOrigin-RevId: 173586350 --- .../contrib/eager/python/metrics_impl.py | 67 ++++++++++++------- .../contrib/eager/python/metrics_test.py | 9 +++ 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 6af0d65e08..6c55ac2f5d 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -38,9 +38,30 @@ _to_replace = re.compile("[^A-Za-z0-9.]") class Metric(object): """A metric holds state for aggregating statistics over an evaluation run. - Users will use Evaluator.add_metric() to add Metric objects to their - evaluation, call them in each step (treating the object as a callable), - and then use Evaluator.all_metric_results() at the end. + Example use with eager execution: + + ```python + m = SomeMetric(...) + for input in ...: + m(input) + print(m.result()) + ``` + + Example use with graph execution: + + ```python + m = SomeMetric(...) + m_placeholder = tf.placeholder(...) + m_update = m(m_placeholder) + # Variables defined in first call, so get the initialization op afterwards. + m_init = m.init_variables() # or tf.global_variables_initializer() + m_result = m.result() + with tf.Session() as sess: + sess.run(m_init) + for input in ...: + sess.run(m_update, feed_dict={m_placeholder: input}) + print(sess.run(m_result)) + ``` Descendants will implement: * `build()`: All variables should be created in this method, by calling @@ -52,18 +73,16 @@ class Metric(object): * `result()`: Computes and returns a final value for the metric from the variables in `self`. - Decendants may override, but usually won't need to: - * `aggregate()`: Adds in the state from a list of metrics of the same type - as `self`. (Default is to sum all the variables.) - * `reset()`: Reset all variables to their initial state. (Default is to - zero all the variables.) - Note that users should not call `aggregate()` or `reset()`, they are for - use by TensorFlow infrastructure. + Decendants may override `aggregate()`, but usually won't need to. It + adds in the state from a list of metrics of the same type as `self`. + (Default is to sum all the variables.) Note that users should not call + `aggregate()`, it is for use by TensorFlow infrastructure. """ def __init__(self, name=None): self._built = False self._vars = [] + self._initial_values = {} self._updates = [] name = name or self.__class__.__name__ # Replace things like spaces in name to create a valid scope name. @@ -109,16 +128,22 @@ class Metric(object): return self._vars def init_variables(self): - """Return an op for initializing this Metric's variables. + """Initializes this Metric's variables. - Only for graph execution. Should be called after variables are created - in the first execution of __call__(). + Should be called after variables are created in the first execution + of `__call__()`. If using graph execution, the return value should be + `run()` in a session before running the op returned by `__call__()`. + (See example above.) Returns: - An op to run. + If using graph execution, this returns an op to perform the + initialization. Under eager execution, the variables are reset to their + initial values as a side effect and this function returns None. """ - assert context.in_graph_mode() - return control_flow_ops.group([v.initializer for v in self._vars]) + if context.in_graph_mode(): + return control_flow_ops.group([v.initializer for v in self._vars]) + for v in self._vars: + v.assign(self._initial_values[v]) # ---- To be implemented by descendants --- def build(self, *args, **kwargs): @@ -193,14 +218,6 @@ class Metric(object): self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics])) # pylint: enable=protected-access - def reset(self): - """Reset this metric to a freshly initialized state. - - Default implementation zeros all the metric variables. - """ - for v in self._vars: - v.assign(math_ops.zeros_like(v)) - # ---- For use by descendants --- def add_variable(self, name, shape=None, dtype=None, initializer=None): """***Only for use by descendants of Metric***.""" @@ -209,6 +226,8 @@ class Metric(object): v = variable_scope.get_variable(name, shape, dtype, initializer, trainable=False, use_resource=True) self._vars.append(v) + if context.in_eager_mode(): + self._initial_values[v] = v.value() return v diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 3ecbaeae69..a8377a0660 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -44,6 +44,15 @@ class MetricsTest(test.TestCase): self.assertEqual(dtypes.float64, m.dtype) self.assertEqual(dtypes.float64, m.result().dtype) + def testInitVariables(self): + m = metrics.Mean() + m([1, 10, 100, 1000]) + m([10000.0, 100000.0]) + self.assertEqual(111111.0/6, m.result().numpy()) + m.init_variables() + m(7) + self.assertEqual(7.0, m.result().numpy()) + def testWriteSummaries(self): m = metrics.Mean() m([1, 10, 100]) -- GitLab From 68b00b0be7e356ee30bf86a8a4a7807fa736dd6b Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Thu, 26 Oct 2017 14:15:17 -0700 Subject: [PATCH 417/573] TFE: Add compatibility doc strings to functional layers PiperOrigin-RevId: 173589146 --- tensorflow/python/layers/convolutional.py | 24 +++++++++++++++++++++++ tensorflow/python/layers/core.py | 8 ++++++++ tensorflow/python/layers/maxout.py | 4 ++++ tensorflow/python/layers/normalization.py | 5 +++++ tensorflow/python/layers/pooling.py | 24 +++++++++++++++++++++++ 5 files changed, 65 insertions(+) diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 6b371c618f..c9bfafaee1 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -386,6 +386,10 @@ def conv1d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Conv1D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -593,6 +597,10 @@ def conv2d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Conv2D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -802,6 +810,10 @@ def conv3d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Conv3D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -1128,6 +1140,10 @@ def separable_conv2d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.SeparableConv2d` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -1430,6 +1446,10 @@ def conv2d_transpose(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Conv2DTranspose` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -1748,6 +1768,10 @@ def conv3d_transpose(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Conv3DTranspose` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index 457bee5cff..b30e5f2074 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -234,6 +234,10 @@ def dense( Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Dense` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -343,6 +347,10 @@ def dropout(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.Dropout` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( diff --git a/tensorflow/python/layers/maxout.py b/tensorflow/python/layers/maxout.py index fa6c8cee97..61cfd7f45c 100644 --- a/tensorflow/python/layers/maxout.py +++ b/tensorflow/python/layers/maxout.py @@ -50,6 +50,10 @@ def maxout(inputs, num_units, axis=-1, name=None): Raises: ValueError: if num_units is not multiple of number of features. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.MaxOut` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index 899be08020..5997d652aa 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -720,6 +720,11 @@ def batch_normalization(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.BatchNormalization` + instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py index ec02ab032d..b3535c4410 100644 --- a/tensorflow/python/layers/pooling.py +++ b/tensorflow/python/layers/pooling.py @@ -148,6 +148,10 @@ def average_pooling1d(inputs, pool_size, strides, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.AveragePooling1D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -217,6 +221,10 @@ def max_pooling1d(inputs, pool_size, strides, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.MaxPooling1D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -362,6 +370,10 @@ def average_pooling2d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.AveragePooling2D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -434,6 +446,10 @@ def max_pooling2d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.MaxPooling2D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -592,6 +608,10 @@ def average_pooling3d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.AveragePooling3D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( @@ -668,6 +688,10 @@ def max_pooling3d(inputs, Raises: ValueError: if eager execution is enabled. + + @compatibility(eager) + Not compatible with eager execution. Use `tf.layers.MaxPooling3D` instead. + @end_compatibility """ if context.in_eager_mode(): raise ValueError( -- GitLab From 2cbe1ffd23a0214492d182935212fc7c613e4a05 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 26 Oct 2017 14:16:10 -0700 Subject: [PATCH 418/573] [XLA] Comment fixes (spelling, grammar, and a bit of correctness). PiperOrigin-RevId: 173589267 --- tensorflow/compiler/xla/array.h | 2 +- tensorflow/compiler/xla/service/algebraic_simplifier.cc | 6 ++++-- tensorflow/compiler/xla/service/buffer_assignment_test.cc | 4 ++-- tensorflow/compiler/xla/service/hlo_constant_folding.cc | 4 ++-- tensorflow/compiler/xla/service/hlo_instruction.cc | 8 ++++---- .../compiler/xla/service/tuple_points_to_analysis.h | 4 ++-- tensorflow/compiler/xla/tests/reduce_test.cc | 8 ++++---- 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h index 3be7060a83..2aedafb91f 100644 --- a/tensorflow/compiler/xla/array.h +++ b/tensorflow/compiler/xla/array.h @@ -302,7 +302,7 @@ class Array { } // Advances the specified set of indexes and returns true if we haven't - // wrapped around (i.e. result isnt {0, 0, ...}). + // wrapped around (i.e. result isn't {0, 0, ...}). bool next_index(std::vector* index) const { CHECK_EQ(index->size(), sizes_.size()); for (int64 i = sizes_.size() - 1; i >= 0; --i) { diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index ae26cc2d99..35ab4d89cc 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -1941,7 +1941,7 @@ Status AlgebraicSimplifierVisitor::HandleWhile(HloInstruction* while_op) { return Status::OK(); } - // Remove while loops with static trip count of 1. + // Remove while loops with static trip count of 0. optional trip_count = GetLoopTripCount(while_op); if (trip_count && *trip_count == 0) { // The loop never executes, so the value of the loop is the value of its @@ -1956,8 +1956,10 @@ Status AlgebraicSimplifierVisitor::HandleWhile(HloInstruction* while_op) { changed_ = true; return Status::OK(); } + + // Transform while loops with static trip count of 1 into a call op, then + // inline the call. if (trip_count && *trip_count == 1) { - // Transform the while loop into a call op, then inline the call. auto computation = while_op->parent(); auto call_op = computation->AddInstruction(HloInstruction::CreateCall( while_op->shape(), while_op->operands(), while_op->while_body())); diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc index e3378a756b..89410f42bd 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc @@ -1179,7 +1179,7 @@ TEST_F(BufferAssignmentTest, TupleCallAsOutput) { auto assignment = RunBufferAssignment(module.get()); EXPECT_EQ(3, assignment->Allocations().size()); - // Buffers for call are co-located with the sub-computation. + // Buffers for call are colocated with the sub-computation. EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{}), GetAllocation(*assignment, sub_tuple, /*index=*/{})); EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{0}), @@ -1238,7 +1238,7 @@ TEST_F(BufferAssignmentTest, TupleChainedCallAsOutput) { auto assignment = RunBufferAssignment(module.get()); - // Buffers for call are co-located with the sub-computations. + // Buffers for call are colocated with the sub-computations. EXPECT_EQ(GetAllocation(*assignment, a_call, /*index=*/{}), GetAllocation(*assignment, b_call, /*index=*/{})); EXPECT_EQ(GetAllocation(*assignment, b_call, /*index=*/{}), diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc index c05bbeb5c9..53450991b6 100644 --- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc +++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc @@ -49,8 +49,8 @@ StatusOr HloConstantFolding::Run(HloModule* module) { continue; } // Skip Constant, Parameter, Reduce operation. - // TODO(b/35975797): Enable Reduce operation once arbitary computation are - // supported by the evaluator. + // TODO(b/35975797): Enable Reduce operation once arbitrary computation + // are supported by the evaluator. // TODO(b/64407269): Enable Tuple once the timeout issue is resolved. if (instruction->opcode() == HloOpcode::kParameter || instruction->opcode() == HloOpcode::kConstant || diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index d53ac221d1..272f573623 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2665,10 +2665,10 @@ class HloInstruction::FusionReusesParamElements { public: using UseKind = HloInstruction::UseKind; - // We could rather iterate backwards thru fused_instructions_ here, as it is - // in reverse postorder, and compute whether each fused instruction reuses - // the value of this parameter, which would save stack space but not allow - // us to finish early if we find a reuse. + // We could rather iterate backwards through fused_instructions_ here, as it + // is in reverse postorder, and compute whether each fused instruction reuses + // the value of this parameter, which would save stack space but not allow us + // to finish early if we find a reuse. static UseKind Compute(int64 i, const HloInstruction& hlo) { tensorflow::gtl::FlatMap memoization_cache; return ComputeInternal(i, hlo, &memoization_cache); diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h index be45732952..30dabb56bd 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h @@ -44,7 +44,7 @@ namespace xla { // A class describing the source(s) of the Buffer(s) contained in the output of // a particular HLO instruction. The structure of PointsToSet mirrors the -// structure of the instruction's shape which may be an arbitrary tree (eg, a +// structure of the instruction's shape, which may be an arbitrary tree (eg, a // nested tuple). Each node in this tree corresponds to a single buffer in the // instruction's output and contains the set of Buffers which might define // the corresponding buffer. @@ -148,7 +148,7 @@ class PointsToSet { ShapeTree tree_; // PointsToSet contains references (const LogicalBuffer*) to elements within - // TuplePointsToAnalysis so disable copying. + // TuplePointsToAnalysis, so disable copying. TF_DISALLOW_COPY_AND_ASSIGN(PointsToSet); }; diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc index 794e5a4920..7bc3185c36 100644 --- a/tensorflow/compiler/xla/tests/reduce_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_test.cc @@ -502,8 +502,8 @@ XLA_TEST_F(ReduceTest, AddReduce2DScalarToR0) { ComputationBuilder builder(client_, TestName()); auto add = CreateScalarAddComputation(F32, &builder); auto scalar = builder.ConstantR0(42.0); - auto broacasted = builder.Broadcast(scalar, {500, 500}); - builder.Reduce(broacasted, builder.ConstantR0(0.0f), add, {0, 1}); + auto broadcasted = builder.Broadcast(scalar, {500, 500}); + builder.Reduce(broadcasted, builder.ConstantR0(0.0f), add, {0, 1}); float expected = 42.0f * static_cast(500 * 500); ComputeAndCompareR0(&builder, expected, {}, ErrorSpec(0.0001)); @@ -514,8 +514,8 @@ XLA_TEST_F(ReduceTest, MaxReduce2DScalarToR0) { ComputationBuilder builder(client_, TestName()); auto max = CreateScalarMaxComputation(F32, &builder); auto scalar = builder.ConstantR0(42.0); - auto broacasted = builder.Broadcast(scalar, {500, 500}); - builder.Reduce(broacasted, builder.ConstantR0(0.0f), max, {0, 1}); + auto broadcasted = builder.Broadcast(scalar, {500, 500}); + builder.Reduce(broadcasted, builder.ConstantR0(0.0f), max, {0, 1}); float expected = 42.0f; ComputeAndCompareR0(&builder, expected, {}, ErrorSpec(0.0001)); -- GitLab From a8e1c3a88d5470e813de9a3642c1bf2f08baef5f Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 26 Oct 2017 14:34:55 -0700 Subject: [PATCH 419/573] [XLA:CPU] [XLA:GPU] Run DCE during fixed-point simplification pass. PiperOrigin-RevId: 173592048 --- tensorflow/compiler/xla/service/cpu/cpu_compiler.cc | 1 + tensorflow/compiler/xla/service/gpu/gpu_compiler.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 99b5035c2d..65e117e68f 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -282,6 +282,7 @@ Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) { [](const Shape&, const Shape&) { return false; }, /*enable_dot_simplification=*/false); pass.AddPass(); + pass.AddPass(); pass.AddPass(); pass.AddPass(); } diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 9c7ca9ea38..b5331fe4e2 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -151,6 +151,7 @@ tensorflow::Status OptimizeHloModule( /*is_layout_sensitive=*/false, [](const Shape&, const Shape&) { return false; }); pass.AddPass(); + pass.AddPass(); pass.AddPass(); pass.AddPass(); } -- GitLab From 87a4991bfba80bec04946330620c35392dc4bbd8 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 26 Oct 2017 14:37:54 -0700 Subject: [PATCH 420/573] Implements count_up_to for resource variables. PiperOrigin-RevId: 173592524 --- tensorflow/core/kernels/BUILD | 2 +- tensorflow/core/kernels/count_up_to_op.cc | 55 +++++++++++++++++-- tensorflow/core/ops/state_ops.cc | 33 +++++++++++ .../resource_variable_ops_test.py | 15 +++++ .../python/ops/resource_variable_ops.py | 24 ++++++++ tensorflow/python/ops/state_ops.py | 22 ++++++++ 6 files changed, 146 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 277b21f833..a3452f2f8c 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -3818,7 +3818,7 @@ STATE_DEPS = [ tf_kernel_library( name = "count_up_to_op", prefix = "count_up_to_op", - deps = STATE_DEPS, + deps = STATE_DEPS + [":variable_ops"], ) tf_kernel_library( diff --git a/tensorflow/core/kernels/count_up_to_op.cc b/tensorflow/core/kernels/count_up_to_op.cc index 040c40d606..9da0015fa2 100644 --- a/tensorflow/core/kernels/count_up_to_op.cc +++ b/tensorflow/core/kernels/count_up_to_op.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/variable_ops.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" @@ -54,10 +55,56 @@ class CountUpToOp : public OpKernel { T limit_; }; -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("CountUpTo").TypeConstraint("T").Device(DEVICE_CPU), \ - CountUpToOp) +template +class ResourceCountUpToOp : public OpKernel { + public: + explicit ResourceCountUpToOp(OpKernelConstruction* context) + : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("limit", &limit_)); + OP_REQUIRES_OK(context, context->GetAttr("T", &dtype_)); + } + + void Compute(OpKernelContext* context) override { + Var* variable = nullptr; + OP_REQUIRES_OK( + context, + LookupResource(context, HandleFromInput(context, 0), &variable)); + core::ScopedUnref s(variable); + mutex_lock l(*variable->mu()); + Tensor before_increment = *variable->tensor(); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(before_increment.shape()), + errors::InvalidArgument("input is not a scalar: ", + before_increment.shape().DebugString())); + if (before_increment.scalar()() >= limit_) { + context->SetStatus(errors::OutOfRange("Reached limit of ", limit_)); + return; + } + // Allocate new buffer + AllocatorAttributes attr; + attr.set_gpu_compatible(true); + attr.set_nic_compatible(true); + PersistentTensor unused; + Tensor* tmp; + OP_REQUIRES_OK(context, context->allocate_persistent( + dtype_, TensorShape({}), &unused, &tmp, attr)); + *variable->tensor() = *tmp; + tmp->scalar()() = before_increment.scalar()() + 1; + context->set_output(0, before_increment); + } + + private: + T limit_; + DataType dtype_; +}; + +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("CountUpTo").TypeConstraint("T").Device(DEVICE_CPU), \ + CountUpToOp) \ + REGISTER_KERNEL_BUILDER( \ + Name("ResourceCountUpTo").TypeConstraint("T").Device(DEVICE_CPU), \ + ResourceCountUpToOp) REGISTER(int32); REGISTER(int64); diff --git a/tensorflow/core/ops/state_ops.cc b/tensorflow/core/ops/state_ops.cc index b86c0b3990..da5f091e9f 100644 --- a/tensorflow/core/ops/state_ops.cc +++ b/tensorflow/core/ops/state_ops.cc @@ -648,4 +648,37 @@ output: A copy of the input before increment. If nothing else modifies the input, the values produced will all be distinct. )doc"); +REGISTER_OP("ResourceCountUpTo") + .Input("resource: resource") + .Output("output: T") + .Attr("limit: int") + .Attr("T: {int32, int64}") + .SetShapeFn([](InferenceContext* c) { + auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data == nullptr || handle_data->empty()) { + return errors::InvalidArgument("Handle has no shape/type information."); + } + shape_inference::ShapeAndType shape_and_type = (*handle_data)[0]; + DataType value_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("T", &value_dtype)); + if (value_dtype != shape_and_type.dtype) { + return errors::InvalidArgument( + "Data types do not match: ", DataTypeString(value_dtype), " and ", + DataTypeString(shape_and_type.dtype)); + } + ShapeHandle output; + TF_RETURN_IF_ERROR(c->WithRank(shape_and_type.shape, 0, &output)); + c->set_output(0, output); + return Status::OK(); + }) + .Doc(R"doc( +Increments variable pointed to by 'resource' until it reaches 'limit'. + +resource: Should be from a scalar `Variable` node. +limit: If incrementing ref would bring it above limit, instead generates an + 'OutOfRange' error. +output: A copy of the input before increment. If nothing else modifies the + input, the values produced will all be distinct. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 32edc5be7f..d8d1ba6bbc 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -30,6 +30,7 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -189,6 +190,20 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): resource_variable_ops.ResourceVariable( 1.0, name="handle-numpy").handle.numpy() + def testCountUpTo(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable(0, name="upto") + self.assertAllEqual(v.count_up_to(1), 0) + with self.assertRaises(errors.OutOfRangeError): + v.count_up_to(1) + + def testCountUpToFunction(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable(0, name="upto") + self.assertAllEqual(state_ops.count_up_to(v, 1), 0) + with self.assertRaises(errors.OutOfRangeError): + state_ops.count_up_to(v, 1) + @test_util.run_in_graph_and_eager_modes() def testInitFnDtype(self): v = resource_variable_ops.ResourceVariable( diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 439fa84238..ee8dd08c43 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_resource_variable_ops +from tensorflow.python.ops import gen_state_ops from tensorflow.python.ops import variables # go/tf-wildcard-import # pylint: disable=wildcard-import @@ -573,6 +574,29 @@ class ResourceVariable(variables.Variable): "numpy() is only available when eager execution is enabled.") return self.read_value().numpy() + def count_up_to(self, limit): + """Increments this variable until it reaches `limit`. + + When that Op is run it tries to increment the variable by `1`. If + incrementing the variable would bring it above `limit` then the Op raises + the exception `OutOfRangeError`. + + If no error is raised, the Op outputs the value of the variable before + the increment. + + This is essentially a shortcut for `count_up_to(self, limit)`. + + Args: + limit: value at which incrementing the variable raises an error. + + Returns: + A `Tensor` that will hold the variable value before the increment. If no + other Op modifies this variable, the values produced will all be + distinct. + """ + return gen_state_ops.resource_count_up_to(self.handle, limit=limit, + T=self.dtype) + def _set_save_slice_info(self, save_slice_info): """Sets the slice info for this `ResourceVariable`. diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py index 65ec2d4b77..5b9ca7c0b9 100644 --- a/tensorflow/python/ops/state_ops.py +++ b/tensorflow/python/ops/state_ops.py @@ -275,3 +275,25 @@ def assign(ref, value, validate_shape=None, use_locking=None, name=None): ref, value, use_locking=use_locking, name=name, validate_shape=validate_shape) return ref.assign(value) + + +def count_up_to(ref, limit, name=None): + r"""Increments 'ref' until it reaches 'limit'. + + Args: + ref: A Variable. Must be one of the following types: `int32`, `int64`. + Should be from a scalar `Variable` node. + limit: An `int`. + If incrementing ref would bring it above limit, instead generates an + 'OutOfRange' error. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `ref`. + A copy of the input before increment. If nothing else modifies the + input, the values produced will all be distinct. + """ + if ref.dtype._is_ref_dtype: + return gen_state_ops.count_up_to(ref, limit=limit, name=name) + return gen_state_ops.resource_count_up_to( + ref.handle, limit, T=ref.dtype, name=name) -- GitLab From 06979c5a27d9cad343028eae32996a68a79e5169 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 14:42:45 -0700 Subject: [PATCH 421/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173593222 --- tensorflow/go/op/wrappers.go | 242 +++++++++++++++++++---------------- 1 file changed, 134 insertions(+), 108 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index cc8165e2c7..615c386858 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -13466,67 +13466,6 @@ func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) { return op.Output(0) } -// Encode audio data using the WAV file format. -// -// This operation will generate a string suitable to be saved out to create a .wav -// audio file. It will be encoded in the 16-bit PCM format. It takes in float -// values in the range -1.0f to 1.0f, and any outside that value will be clamped to -// that range. -// -// `audio` is a 2-D float Tensor of shape `[length, channels]`. -// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100). -// -// Arguments: -// audio: 2-D with shape `[length, channels]`. -// sample_rate: Scalar containing the sample frequency. -// -// Returns 0-D. WAV-encoded file contents. -func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "EncodeWav", - Input: []tf.Input{ - audio, sample_rate, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// The gradient operator for the SparseAdd op. -// -// The SparseAdd op calculates A + B, where A, B, and the sum are all represented -// as `SparseTensor` objects. This op takes in the upstream gradient w.r.t. -// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty -// values of A and B. -// -// Arguments: -// backprop_val_grad: 1-D with shape `[nnz(sum)]`. The gradient with respect to -// the non-empty values of the sum. -// a_indices: 2-D. The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`. -// b_indices: 2-D. The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`. -// sum_indices: 2-D. The `indices` of the sum `SparseTensor`, size -// `[nnz(sum), ndims]`. -// -// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the -// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the -// non-empty values of B. -func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SparseAddGrad", - Input: []tf.Input{ - backprop_val_grad, a_indices, b_indices, sum_indices, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - // Read an element from the TensorArray into output `value`. // // Arguments: @@ -14567,6 +14506,32 @@ func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) { return op.Output(0) } +// Increments variable pointed to by 'resource' until it reaches 'limit'. +// +// Arguments: +// resource: Should be from a scalar `Variable` node. +// limit: If incrementing ref would bring it above limit, instead generates an +// 'OutOfRange' error. +// +// +// Returns A copy of the input before increment. If nothing else modifies the +// input, the values produced will all be distinct. +func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"limit": limit, "T": T} + opspec := tf.OpSpec{ + Type: "ResourceCountUpTo", + Input: []tf.Input{ + resource, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Looks up keys in a table, outputs the corresponding values. // // The tensor `keys` must of the same type as the keys of the table. @@ -17320,6 +17285,53 @@ func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Outp return op.Output(0) } +// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad. +type ResourceSparseApplyAdagradAttr func(optionalAttr) + +// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value. +// +// value: If `True`, updating of the var and accum tensors will be protected +// by a lock; otherwise the behavior is undefined, but may exhibit less +// contention. +// If not specified, defaults to false +func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// Update relevant entries in '*var' and '*accum' according to the adagrad scheme. +// +// That is for rows we have grad for, we update var and accum as follows: +// accum += grad * grad +// var -= lr * grad * (1 / sqrt(accum)) +// +// Arguments: +// var_: Should be from a Variable(). +// accum: Should be from a Variable(). +// lr: Learning rate. Must be a scalar. +// grad: The gradient. +// indices: A vector of indices into the first dimension of var and accum. +// +// Returns the created operation. +func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResourceSparseApplyAdagrad", + Input: []tf.Input{ + var_, accum, lr, grad, indices, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + // Creates a dataset that zips together `input_datasets`. func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { if scope.Err() != nil { @@ -17514,53 +17526,6 @@ func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) return op.Output(0) } -// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad. -type ResourceSparseApplyAdagradAttr func(optionalAttr) - -// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value. -// -// value: If `True`, updating of the var and accum tensors will be protected -// by a lock; otherwise the behavior is undefined, but may exhibit less -// contention. -// If not specified, defaults to false -func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr { - return func(m optionalAttr) { - m["use_locking"] = value - } -} - -// Update relevant entries in '*var' and '*accum' according to the adagrad scheme. -// -// That is for rows we have grad for, we update var and accum as follows: -// accum += grad * grad -// var -= lr * grad * (1 / sqrt(accum)) -// -// Arguments: -// var_: Should be from a Variable(). -// accum: Should be from a Variable(). -// lr: Learning rate. Must be a scalar. -// grad: The gradient. -// indices: A vector of indices into the first dimension of var and accum. -// -// Returns the created operation. -func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "ResourceSparseApplyAdagrad", - Input: []tf.Input{ - var_, accum, lr, grad, indices, - }, - Attrs: attrs, - } - return scope.AddOperation(opspec) -} - // AvgPool3DGradAttr is an optional argument to AvgPool3DGrad. type AvgPool3DGradAttr func(optionalAttr) @@ -17937,6 +17902,67 @@ func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64, return op.Output(0) } +// Encode audio data using the WAV file format. +// +// This operation will generate a string suitable to be saved out to create a .wav +// audio file. It will be encoded in the 16-bit PCM format. It takes in float +// values in the range -1.0f to 1.0f, and any outside that value will be clamped to +// that range. +// +// `audio` is a 2-D float Tensor of shape `[length, channels]`. +// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100). +// +// Arguments: +// audio: 2-D with shape `[length, channels]`. +// sample_rate: Scalar containing the sample frequency. +// +// Returns 0-D. WAV-encoded file contents. +func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "EncodeWav", + Input: []tf.Input{ + audio, sample_rate, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// The gradient operator for the SparseAdd op. +// +// The SparseAdd op calculates A + B, where A, B, and the sum are all represented +// as `SparseTensor` objects. This op takes in the upstream gradient w.r.t. +// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty +// values of A and B. +// +// Arguments: +// backprop_val_grad: 1-D with shape `[nnz(sum)]`. The gradient with respect to +// the non-empty values of the sum. +// a_indices: 2-D. The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`. +// b_indices: 2-D. The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`. +// sum_indices: 2-D. The `indices` of the sum `SparseTensor`, size +// `[nnz(sum), ndims]`. +// +// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the +// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the +// non-empty values of B. +func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SparseAddGrad", + Input: []tf.Input{ + backprop_val_grad, a_indices, b_indices, sum_indices, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + // FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2. type FixedLengthRecordReaderV2Attr func(optionalAttr) -- GitLab From de2c876843cdb86482af64bd3b39eadc7ab8cff5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 15:09:01 -0700 Subject: [PATCH 422/573] Added max_folded_constant_in_bytes field in OptimizationConfig. This option controls the maximum constant node size created during constant folding optimization. The default value is 10 MiB, the same limit as before this change. PiperOrigin-RevId: 173597212 --- .../core/common_runtime/constant_folding.cc | 16 +++++++++------- .../core/common_runtime/constant_folding.h | 3 +++ .../core/common_runtime/constant_folding_test.cc | 7 +++++++ .../core/common_runtime/graph_optimizer.cc | 4 ++++ tensorflow/core/protobuf/config.proto | 7 +++++++ .../golden/tensorflow.-optimizer-options.pbtxt | 4 ++++ 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/common_runtime/constant_folding.cc b/tensorflow/core/common_runtime/constant_folding.cc index aca68d4c4a..0398c2a60d 100644 --- a/tensorflow/core/common_runtime/constant_folding.cc +++ b/tensorflow/core/common_runtime/constant_folding.cc @@ -460,7 +460,8 @@ Graph* GetConstantGraph( // new constant node. bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device, NodeAndOutput tensor, const Tensor& constant, - const gtl::FlatSet& control_deps) { + const gtl::FlatSet& control_deps, + int64 max_constant_size_in_bytes) { // Be conservative when replacing a tensor with a constant, when not // running on CPU. // 1) If the destination tensor is not an int32 tensor, and has HOST_MEMORY @@ -469,8 +470,9 @@ bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device, // constraint, do not replace it. // 3) If the constant op created does not have a kernel implementation // for the device, do not use it. - // 4) If the size of the constant in bytes is too large (> 10M), do not - // replace it. This prevents the size of the Graph from growing too large. + // 4) If the size of the constant in bytes is too large (> + // max_constant_in_bytes), do not replace it. This prevents the size of the + // Graph from growing too large. // TODO(keveman): Consider adding a new constant op that has a kernel // implementation for all types, but with HostMemory constraint on it's // output. @@ -494,7 +496,7 @@ bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device, return false; } } - if (constant.TotalBytes() > 10 * 1024 * 1024) { + if (constant.TotalBytes() > max_constant_size_in_bytes) { return false; } @@ -613,9 +615,9 @@ Status ConstantFold(const ConstantFoldingOptions& opts, for (size_t c = 0; c < outputs.size(); ++c) { const gtl::FlatSet& control_deps = constant_control_deps[tensors_to_replace[c].first]; - if (ReplaceTensorWithConstant(graph, partition_device, - tensors_to_replace[c], outputs[c], - control_deps)) { + if (ReplaceTensorWithConstant( + graph, partition_device, tensors_to_replace[c], outputs[c], + control_deps, opts.max_constant_size_in_bytes)) { ++num_nodes_replaced; } } diff --git a/tensorflow/core/common_runtime/constant_folding.h b/tensorflow/core/common_runtime/constant_folding.h index e7b1571a81..e4d724c58a 100644 --- a/tensorflow/core/common_runtime/constant_folding.h +++ b/tensorflow/core/common_runtime/constant_folding.h @@ -34,6 +34,9 @@ struct ConstantFoldingOptions { // outputs. const std::unordered_map>* shape_map = nullptr; // not owned + // The maximum size of each constant created during constant folding + // optimization. + int64 max_constant_size_in_bytes = 10 * 1024 * 1024; }; // Perform constant folding optimization on "graph". diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc index 2c7c20817a..923a4d9249 100644 --- a/tensorflow/core/common_runtime/constant_folding_test.cc +++ b/tensorflow/core/common_runtime/constant_folding_test.cc @@ -259,6 +259,13 @@ TEST_F(ConstantFoldingTest, TestNoReplaceLargeConstant) { TF_EXPECT_OK(ConstantFold(ConstantFoldingOptions{}, nullptr, Env::Default(), nullptr, &g, &was_mutated)); EXPECT_FALSE(was_mutated); + + // Increase the limit and the concat should now be constant folded. + ConstantFoldingOptions opt; + opt.max_constant_size_in_bytes = 10 * 1024 * 1024 + 4; + TF_EXPECT_OK( + ConstantFold(opt, nullptr, Env::Default(), nullptr, &g, &was_mutated)); + EXPECT_TRUE(was_mutated); } TEST_F(ConstantFoldingTest, TestNoReplaceFunctionCall) { diff --git a/tensorflow/core/common_runtime/graph_optimizer.cc b/tensorflow/core/common_runtime/graph_optimizer.cc index ff99db9532..def185e522 100644 --- a/tensorflow/core/common_runtime/graph_optimizer.cc +++ b/tensorflow/core/common_runtime/graph_optimizer.cc @@ -61,6 +61,10 @@ void GraphOptimizer::Optimize( if (opts_.do_constant_folding()) { ConstantFoldingOptions cf_opts; cf_opts.shape_map = shape_map; + if (opts_.max_folded_constant_in_bytes() > 0) { + cf_opts.max_constant_size_in_bytes = + opts_.max_folded_constant_in_bytes(); + } bool was_mutated; ConstantFold(cf_opts, runtime, env, device, g, &was_mutated) .IgnoreError(); diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto index 56bb709e11..145311b59d 100644 --- a/tensorflow/core/protobuf/config.proto +++ b/tensorflow/core/protobuf/config.proto @@ -87,6 +87,13 @@ message OptimizerOptions { // If true, perform constant folding optimization on the graph. bool do_constant_folding = 2; + // Constant folding optimization replaces tensors whose values can be + // predetermined, with constant nodes. To avoid inserting too large constants, + // the size of each constant created can be limited. If this value is zero, a + // default limit of 10 MiB will be applied. If constant folding optimization + // is disabled, this value is ignored. + int64 max_folded_constant_in_bytes = 6; + // If true, perform function inlining on the graph. bool do_function_inlining = 4; diff --git a/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt b/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt index 5dd1ee47c9..6cac5c4d99 100644 --- a/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt @@ -46,6 +46,10 @@ tf_class { name: "Level" mtype: "" } + member { + name: "MAX_FOLDED_CONSTANT_IN_BYTES_FIELD_NUMBER" + mtype: "" + } member { name: "OFF" mtype: "" -- GitLab From f255bfc2e8a2150f65f899e6301fb2f0d0b84659 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 15:15:02 -0700 Subject: [PATCH 423/573] Use object-oriented metrics in eager MNIST example. PiperOrigin-RevId: 173598122 --- tensorflow/contrib/eager/python/metrics_impl.py | 2 +- tensorflow/contrib/eager/python/tfe.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 6c55ac2f5d..795dff548f 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -263,7 +263,7 @@ class Mean(Metric): """ if weights is None: self.denom.assign_add( - math_ops.cast(array_ops.size(values), self.dtype)) + math_ops.cast(array_ops.identity(array_ops.size(values)), self.dtype)) values = math_ops.reduce_sum(values) self.numer.assign_add(math_ops.cast(values, self.dtype)) else: diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 3810d96950..a769140713 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -18,6 +18,8 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice. To use, at program startup, call `tfe.enable_eager_execution()`. +@@metrics + @@list_devices @@num_gpus @@ -68,6 +70,7 @@ from __future__ import print_function # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import # +from tensorflow.contrib.eager.python import metrics from tensorflow.contrib.eager.python.datasets import Iterator from tensorflow.contrib.eager.python.network import Network from tensorflow.contrib.eager.python.saver import restore_variables_on_create -- GitLab From a0c33c48e05a32adddf2119cc20bc3ea7a0e7253 Mon Sep 17 00:00:00 2001 From: Oleg Zabluda Date: Thu, 26 Oct 2017 15:19:22 -0700 Subject: [PATCH 424/573] Fix documentation error in tf.size() tf.size() returns a symbolic `Tensor`, not an integer. --- tensorflow/python/ops/array_ops.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 97dc63ebb1..857cd09d56 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -309,8 +309,8 @@ def size(input, name=None, out_type=dtypes.int32): # pylint: disable=redefined-builtin """Returns the size of a tensor. - This operation returns an integer representing the number of elements in - `input`. + Returns a 0-D `Tensor` representing the number of elements in `input` + of type `out_type`. Defaults to tf.int32. For example: @@ -327,6 +327,10 @@ def size(input, name=None, out_type=dtypes.int32): Returns: A `Tensor` of type `out_type`. Defaults to tf.int32. + + @compatibility(numpy) + Equivalent to np.size() + @end_compatibility """ return size_internal(input, name, optimize=True, out_type=out_type) -- GitLab From 7ecbf3501411acb44fd53c9cb54ea741a76f0776 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 15:20:07 -0700 Subject: [PATCH 425/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173598929 --- .../core/ops/compat/ops_history.v1.pbtxt | 26 ++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 30 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index cec75f6799..076c7bea1a 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -27445,6 +27445,32 @@ op { } is_stateful: true } +op { + name: "ResourceCountUpTo" + input_arg { + name: "resource" + type: DT_RESOURCE + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "limit" + type: "int" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + is_stateful: true +} op { name: "ResourceGather" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 78f0fda408..0a590fef00 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -22128,6 +22128,36 @@ op { description: "Note that in dense implementation of this algorithm, ms and mom will\nupdate even if the grad is zero, but in this sparse implementation, ms\nand mom will not update in iterations during which the grad is zero.\n\nmean_square = decay * mean_square + (1-decay) * gradient ** 2\nDelta = learning_rate * gradient / sqrt(mean_square + epsilon)\n\nms <- rho * ms_{t-1} + (1-rho) * grad * grad\nmom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)\nvar <- var - mom" is_stateful: true } +op { + name: "ResourceCountUpTo" + input_arg { + name: "resource" + description: "Should be from a scalar `Variable` node." + type: DT_RESOURCE + } + output_arg { + name: "output" + description: "A copy of the input before increment. If nothing else modifies the\ninput, the values produced will all be distinct." + type_attr: "T" + } + attr { + name: "limit" + type: "int" + description: "If incrementing ref would bring it above limit, instead generates an\n\'OutOfRange\' error." + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + summary: "Increments variable pointed to by \'resource\' until it reaches \'limit\'." + is_stateful: true +} op { name: "ResourceGather" input_arg { -- GitLab From 1ba3cb0645a2216718ae6baabf504cd154d17ad3 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Thu, 26 Oct 2017 15:27:49 -0700 Subject: [PATCH 426/573] Implement saving/restoring state of TextLineDataset. PiperOrigin-RevId: 173600100 --- .../contrib/data/python/kernel_tests/BUILD | 2 + .../kernel_tests/reader_dataset_ops_test.py | 273 ++++++++++++++++++ tensorflow/core/kernels/reader_dataset_ops.cc | 139 ++++++--- 3 files changed, 377 insertions(+), 37 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index b8cdb7b20d..36af55a7ec 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -270,6 +270,7 @@ py_test( srcs = ["reader_dataset_ops_test.py"], srcs_version = "PY2AND3", deps = [ + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:readers", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -283,6 +284,7 @@ py_test( "//tensorflow/python:lib", "//tensorflow/python:parsing_ops", "//tensorflow/python:tensor_shape", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python/data/ops:iterator_ops", ], diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py index 2682e8bdfa..3ae8f71d77 100644 --- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py @@ -21,6 +21,7 @@ import gzip import os import zlib +from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops from tensorflow.contrib.data.python.ops import readers from tensorflow.core.example import example_pb2 from tensorflow.core.example import feature_pb2 @@ -36,6 +37,7 @@ from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import io_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib from tensorflow.python.util import compat @@ -163,6 +165,277 @@ class TextLineDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(iterator.get_next()) + def _ckpt_path(self): + return os.path.join(self.get_temp_dir(), "iterator") + + def _latest_ckpt(self): + return saver_lib.latest_checkpoint(self.get_temp_dir()) + + def _save(self, saver, sess): + saver.save(sess, self._ckpt_path()) + + def _restore(self, saver, sess): + saver.restore(sess, self._latest_ckpt()) + + def _import_meta_graph(self): + meta_file_path = self._ckpt_path() + ".meta" + return saver_lib.import_meta_graph(meta_file_path) + + def _build_graph(self, + test_filenames, + compression_type=None, + build_saveable=True): + ds = readers.TextLineDataset( + test_filenames, compression_type=compression_type, buffer_size=10) + iterator = ds.make_initializable_iterator() + if build_saveable: + saveable = contrib_iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + init_op = iterator.initializer + get_next = iterator.get_next() + ops.add_to_collection("iterator_ops", init_op) + ops.add_to_collection("iterator_ops", get_next) + saver = saver_lib.Saver(allow_empty=True) + return init_op, get_next, saver + + def _testReadWithBreaks(self, breaks, num_files=5, lines_per_file=5): + """Tests reading from input pipeline with regular breaks. + + At each break point the iterator state gets saved using Saver and reloaded + in a new Graph and session. + + Args: + breaks: List of counts of records after reading which iterator state is + checkpointed. Must to in non-decreasing order. + num_files: Total number of files. + lines_per_file: Total number of lines per file. + """ + compression_types = [None, "GZIP", "ZLIB"] + for compression_type in compression_types: + test_filenames = self._createFiles( + num_files, + lines_per_file, + crlf=True, + compression_type=compression_type) + + # Collect ground truth. + total_records = num_files * lines_per_file + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type=compression_type) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(total_records): + expected_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + # Simulate run with breaks. + actual_records = [] + next_record_index = 0 + load_from_ckpt = False + breaks.append(total_records) + for break_index in breaks: + with ops.Graph().as_default() as g: + if not load_from_ckpt: + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type=compression_type) + else: + saver = self._import_meta_graph() + init_op, get_next = ops.get_collection("iterator_ops") + + with self.test_session(graph=g) as sess: + if not load_from_ckpt: + sess.run(init_op) + else: + self._restore(saver, sess) + while next_record_index != break_index: + actual_records.append(sess.run(get_next)) + next_record_index += 1 + if break_index == total_records: + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self._save(saver, sess) + load_from_ckpt = True + self.assertEqual(actual_records, expected_records) + + def testSaveAtFileBoundary(self): + self._testReadWithBreaks([10]) + + def testSaveWithinFile(self): + self._testReadWithBreaks([12]) + + def testSaveUnusedIterator(self): + self._testReadWithBreaks([0]) + + def testSaveRestoreIdempotence(self): + # Attempt to save an iterator immediately after it has been + # restored. + self._testReadWithBreaks([0, 0]) + self._testReadWithBreaks([10, 10]) + self._testReadWithBreaks([12, 12]) + + def testMultipleBreaks(self): + self._testReadWithBreaks([0, 4, 20]) + + def testRestoreExhaustedIterator(self): + num_files = 2 + lines_per_file = 5 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(num_files * lines_per_file): + sess.run(get_next) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self._save(saver, sess) + + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + saver = self._import_meta_graph() + self._restore(saver, sess) + _, get_next = ops.get_collection("iterator_ops") + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testInitThenRestore(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + sess.run(get_next) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + saver = self._import_meta_graph() + init_op, get_next = ops.get_collection("iterator_ops") + sess.run(init_op) + self._restore(saver, sess) + for _ in range(total_records - break_record): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + + def testRestoreInModifiedGraph(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + sess.run(get_next) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type="GZIP") + self._restore(saver, sess) + for _ in range(total_records - break_record): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + + def testRestoreInModifiedGraphThenInit(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + expected_records.append(sess.run(get_next)) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + # Test that calling the init_op overrides the restored iterator. The + # iterator for the old graph was build to read uncompressed files and + # would fail when trying to read the new files. + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + test_filenames = self._createFiles( + num_files, lines_per_file, crlf=True, compression_type="GZIP") + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type="GZIP") + self._restore(saver, sess) + sess.run(init_op) + for _ in range(total_records): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + + def testDoNotRestoreIterator(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + expected_records.append(sess.run(get_next)) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + init_op, get_next, saver = self._build_graph( + test_filenames, build_saveable=False) + self._restore(saver, sess) + with self.assertRaises(errors.FailedPreconditionError): + sess.run(get_next) + sess.run(init_op) + for _ in range(total_records): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + class FixedLengthRecordReaderTest(test.TestCase): diff --git a/tensorflow/core/kernels/reader_dataset_ops.cc b/tensorflow/core/kernels/reader_dataset_ops.cc index fb88c55f73..39ef92a5de 100644 --- a/tensorflow/core/kernels/reader_dataset_ops.cc +++ b/tensorflow/core/kernels/reader_dataset_ops.cc @@ -54,14 +54,9 @@ class TextLineDatasetOp : public DatasetOpKernel { io::ZlibCompressionOptions zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); - bool use_compression = false; - if (compression_type.empty()) { - use_compression = false; - } else if (compression_type == "ZLIB") { - use_compression = true; + if (compression_type == "ZLIB") { zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); } else if (compression_type == "GZIP") { - use_compression = true; zlib_compression_options = io::ZlibCompressionOptions::GZIP(); } else { OP_REQUIRES(ctx, compression_type.empty(), @@ -79,17 +74,20 @@ class TextLineDatasetOp : public DatasetOpKernel { filenames.push_back(filenames_tensor->flat()(i)); } - *output = new Dataset(std::move(filenames), use_compression, + *output = new Dataset(ctx, std::move(filenames), compression_type, zlib_compression_options); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - Dataset(std::vector filenames, bool use_compression, + Dataset(OpKernelContext* ctx, std::vector filenames, + const string& compression_type, const io::ZlibCompressionOptions& options) - : filenames_(std::move(filenames)), - use_compression_(use_compression), + : GraphDatasetBase(ctx), + filenames_(std::move(filenames)), + compression_type_(compression_type), + use_compression_(!compression_type.empty()), options_(options) {} std::unique_ptr MakeIterator( @@ -111,6 +109,21 @@ class TextLineDatasetOp : public DatasetOpKernel { string DebugString() override { return "TextLineDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + Node* filenames = nullptr; + Node* compression_type = nullptr; + Node* buffer_size = nullptr; + TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames)); + TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); + TF_RETURN_IF_ERROR( + b->AddScalar(options_.input_buffer_size, &buffer_size)); + TF_RETURN_IF_ERROR(b->AddDataset( + this, {filenames, compression_type, buffer_size}, output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -123,7 +136,7 @@ class TextLineDatasetOp : public DatasetOpKernel { mutex_lock l(mu_); do { // We are currently processing a file, so try to read the next line. - if (processing_file_) { + if (buffered_input_stream_) { string line_contents; Status s = buffered_input_stream_->ReadLine(&line_contents); @@ -138,14 +151,9 @@ class TextLineDatasetOp : public DatasetOpKernel { // Report non-EOF errors to the caller. return s; } - // We have reached the end of the current file, so maybe // move on to next file. - processing_file_ = false; - input_stream_.reset(); - zlib_input_stream_.reset(); - buffered_input_stream_.reset(); - file_.reset(); + ResetStreamsLocked(); ++current_file_index_; } @@ -155,30 +163,86 @@ class TextLineDatasetOp : public DatasetOpKernel { return Status::OK(); } - // Actually move on to next file. - TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile( - dataset()->filenames_[current_file_index_], &file_)); - processing_file_ = true; - input_stream_.reset( - new io::RandomAccessInputStream(file_.get(), false)); - if (dataset()->use_compression_) { - zlib_input_stream_.reset(new io::ZlibInputStream( - input_stream_.get(), dataset()->options_.input_buffer_size, - dataset()->options_.input_buffer_size, dataset()->options_)); - buffered_input_stream_.reset(new io::BufferedInputStream( - zlib_input_stream_.get(), dataset()->options_.input_buffer_size, - false)); - } else { - buffered_input_stream_.reset(new io::BufferedInputStream( - input_stream_.get(), dataset()->options_.input_buffer_size, - false)); - } + TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); } while (true); } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"), + current_file_index_)); + + // `buffered_input_stream_` is empty if + // 1. GetNext has not been called even once. + // 2. All files have been read and iterator has been exhausted. + if (buffered_input_stream_) { + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name("current_pos"), buffered_input_stream_->Tell())); + } + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + ResetStreamsLocked(); + int64 current_file_index; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"), + ¤t_file_index)); + current_file_index_ = size_t(current_file_index); + // The key "current_pos" is written only if the iterator was saved + // with an open file. + if (reader->Contains(full_name("current_pos"))) { + int64 current_pos; + TF_RETURN_IF_ERROR( + reader->ReadScalar(full_name("current_pos"), ¤t_pos)); + + TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); + TF_RETURN_IF_ERROR(buffered_input_stream_->Seek(current_pos)); + } + return Status::OK(); + } + private: + // Sets up reader streams to read from the file at `current_file_index_`. + Status SetupStreamsLocked(Env* env) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (current_file_index_ >= dataset()->filenames_.size()) { + return errors::InvalidArgument( + "current_file_index_:", current_file_index_, + " >= filenames_.size():", dataset()->filenames_.size()); + } + + // Actually move on to next file. + TF_RETURN_IF_ERROR(env->NewRandomAccessFile( + dataset()->filenames_[current_file_index_], &file_)); + input_stream_.reset( + new io::RandomAccessInputStream(file_.get(), false)); + + if (dataset()->use_compression_) { + zlib_input_stream_.reset(new io::ZlibInputStream( + input_stream_.get(), dataset()->options_.input_buffer_size, + dataset()->options_.input_buffer_size, dataset()->options_)); + buffered_input_stream_.reset(new io::BufferedInputStream( + zlib_input_stream_.get(), dataset()->options_.input_buffer_size, + false)); + } else { + buffered_input_stream_.reset(new io::BufferedInputStream( + input_stream_.get(), dataset()->options_.input_buffer_size, + false)); + } + return Status::OK(); + } + + // Resets all reader streams. + void ResetStreamsLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + input_stream_.reset(); + zlib_input_stream_.reset(); + buffered_input_stream_.reset(); + file_.reset(); + } + mutex mu_; - bool processing_file_ GUARDED_BY(mu_) = false; std::unique_ptr input_stream_ GUARDED_BY(mu_); std::unique_ptr zlib_input_stream_ GUARDED_BY(mu_); @@ -190,6 +254,7 @@ class TextLineDatasetOp : public DatasetOpKernel { }; const std::vector filenames_; + const string compression_type_; const bool use_compression_; const io::ZlibCompressionOptions options_; }; -- GitLab From 029ce4ed70a1d8644a8e1c11345bcae2416a2a6d Mon Sep 17 00:00:00 2001 From: Anush Elangovan Date: Thu, 26 Oct 2017 15:35:12 -0700 Subject: [PATCH 427/573] iOS/RPi Add the ability to choose ANDROID_TYPES_FULL Some networks require "full" types instead of "slim" so remove the hard coding of SLIM in iOS and RPi. It still defaults to building SLIM for them if not ENV var is specified but now you can build with ANDROID_TYPES="-D__ANDROID_TYPES_FULL" \ ./tensorflow/contrib/makefile/build_all_ios.sh TEST: Verify the -D__ANDROID_TYPES_SLIM__ flag is default and you can override with an env var --- tensorflow/contrib/makefile/Makefile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index b582493131..3b4d0ff799 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -44,6 +44,11 @@ ifdef HEXAGON_LIBS endif endif # HEXAGON_LIBS +# If ANDROID_TYPES is not set assume __ANDROID_TYPES_SLIM__ +ifeq ($(ANDROID_TYPES),) + ANDROID_TYPES := -D__ANDROID_TYPES_SLIM__ +endif + # Try to figure out the host system HOST_OS := ifeq ($(OS),Windows_NT) @@ -216,7 +221,7 @@ ifeq ($(TARGET),LINUX) endif # If we're cross-compiling for the Raspberry Pi, use the right gcc. ifeq ($(TARGET),PI) - CXXFLAGS += -D__ANDROID_TYPES_SLIM__ -DRASPBERRY_PI + CXXFLAGS += $(ANDROID_TYPES) -DRASPBERRY_PI LDFLAGS := -Wl,--no-whole-archive LIBS += -ldl -lpthread LIBFLAGS += -Wl,--allow-multiple-definition -Wl,--whole-archive @@ -338,7 +343,7 @@ ifeq ($(TARGET),IOS) -Wno-c++11-narrowing \ -mno-thumb \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONEOS_SYSROOT} @@ -362,7 +367,7 @@ ifeq ($(TARGET),IOS) -Wno-c++11-narrowing \ -mno-thumb \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONEOS_SYSROOT} @@ -385,7 +390,7 @@ ifeq ($(TARGET),IOS) -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONEOS_SYSROOT} @@ -409,7 +414,7 @@ ifeq ($(TARGET),IOS) -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONESIMULATOR_SYSROOT} @@ -432,7 +437,7 @@ ifeq ($(TARGET),IOS) -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONESIMULATOR_SYSROOT} -- GitLab From f13b76a52886e39d8e97c9256c383eb3b79748d8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 15:36:51 -0700 Subject: [PATCH 428/573] Replace the fake updates with no updates when not training. This is possible now that the tf.cond bug has been fixed, and is needed to remove rare data races. PiperOrigin-RevId: 173601427 --- tensorflow/python/layers/normalization.py | 51 +++++++++++++---------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index 5997d652aa..4fbe4b574f 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -436,27 +436,30 @@ class BatchNormalization(base.Layer): if dmax is not None: d = math_ops.maximum(d, -dmax) d = math_ops.minimum(d, dmax) - # When not training, use r=1, d=0, and decay=1 meaning no updates. + # When not training, use r=1, d=0. r = utils.smart_cond(training, lambda: r, lambda: array_ops.ones_like(r)) d = utils.smart_cond(training, lambda: d, lambda: array_ops.zeros_like(d)) - decay = utils.smart_cond(training, lambda: self.renorm_momentum, lambda: 1.) def _update_renorm_variable(var, weight, value): """Updates a moving average and weight, returns the unbiased value.""" - # Update the variables without zero debiasing. The debiasing will be - # accomplished by dividing the exponential moving average by the weight. - # For example, after a single update, the moving average would be - # (1-decay) * value. and the weight will be 1-decay, with their ratio - # giving value. - # Make sure the weight is not updated until before r and d computation. value = array_ops.identity(value) - with ops.control_dependencies([value]): - weight_value = array_ops.constant(1., dtype=weight.dtype) - new_var = moving_averages.assign_moving_average( - var, value, decay, zero_debias=False) - new_weight = moving_averages.assign_moving_average( - weight, weight_value, decay, zero_debias=False) - return new_var / new_weight + def _do_update(): + # Update the variables without zero debiasing. The debiasing will be + # accomplished by dividing the exponential moving average by the weight. + # For example, after a single update, the moving average would be + # (1-decay) * value. and the weight will be 1-decay, with their ratio + # giving the value. + # Make sure the weight is not updated until before r and d computation. + with ops.control_dependencies([value]): + weight_value = array_ops.constant(1., dtype=weight.dtype) + new_var = moving_averages.assign_moving_average( + var, value, self.renorm_momentum, zero_debias=False) + new_weight = moving_averages.assign_moving_average( + weight, weight_value, self.renorm_momentum, zero_debias=False) + return new_var / new_weight + def _fake_update(): + return array_ops.identity(var) + return utils.smart_cond(training, _do_update, _fake_update) with ops.colocate_with(self.moving_mean): new_mean = _update_renorm_variable(self.renorm_mean, @@ -562,8 +565,6 @@ class BatchNormalization(base.Layer): else: new_mean, new_variance = mean, variance - # Update moving averages when training, and prevent updates otherwise. - decay = utils.smart_cond(training, lambda: self.momentum, lambda: 1.) if self.virtual_batch_size is not None: # This isn't strictly correct since in ghost batch norm, you are # supposed to sequentially update the moving_mean and moving_variance @@ -575,10 +576,18 @@ class BatchNormalization(base.Layer): new_variance = math_ops.reduce_mean(new_variance, axis=1, keep_dims=True) - mean_update = moving_averages.assign_moving_average( - self.moving_mean, new_mean, decay, zero_debias=False) - variance_update = moving_averages.assign_moving_average( - self.moving_variance, new_variance, decay, zero_debias=False) + def _do_update(var, value): + return moving_averages.assign_moving_average( + var, value, self.momentum, zero_debias=False) + + mean_update = utils.smart_cond( + training, + lambda: _do_update(self.moving_mean, new_mean), + lambda: self.moving_mean) + variance_update = utils.smart_cond( + training, + lambda: _do_update(self.moving_variance, new_variance), + lambda: self.moving_variance) if context.in_graph_mode(): self.add_update(mean_update, inputs=inputs) self.add_update(variance_update, inputs=inputs) -- GitLab From 9c40507f80434058f600ebebb8b9d6971dd0bdb4 Mon Sep 17 00:00:00 2001 From: Petros Mol Date: Thu, 26 Oct 2017 15:40:44 -0700 Subject: [PATCH 429/573] Documentation and error message edits for third_party/tensorflow/python/estimator/canned/head.py PiperOrigin-RevId: 173601925 --- tensorflow/python/estimator/canned/head.py | 38 ++++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py index f26e54ff49..18806db5eb 100644 --- a/tensorflow/python/estimator/canned/head.py +++ b/tensorflow/python/estimator/canned/head.py @@ -366,11 +366,11 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes, `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. - label_vocabulary: A list of strings represents possible label values. If it - is not given, that means labels are already encoded as integer within - [0, n_classes). If given, labels must be string type and have any value in - `label_vocabulary`. Also there will be errors if vocabulary is not - provided and labels are string. + label_vocabulary: A list or tuple of strings representing possible label + values. If it is not given, that means labels are already encoded as an + integer within [0, n_classes). If given, labels must be of string type and + have any value in `label_vocabulary`. Note that errors will be raised if + `label_vocabulary` is not provided but labels are strings. name: name of the head. If provided, summary and metrics keys will be suffixed by `"/" + name`. Also used as `name_scope` when creating ops. @@ -382,8 +382,9 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes, """ if label_vocabulary is not None and not isinstance(label_vocabulary, (list, tuple)): - raise ValueError('label_vocabulary should be a list. Given type: {}'.format( - type(label_vocabulary))) + raise ValueError( + 'label_vocabulary should be a list or a tuple. Given type: {}'.format( + type(label_vocabulary))) return _MultiClassHeadWithSoftmaxCrossEntropyLoss(n_classes, weight_column, label_vocabulary, name) @@ -437,8 +438,8 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): """Converts labels to integer id space.""" if self._label_vocabulary is None: if not labels.dtype.is_integer: - raise ValueError('Labels dtype should be integer ' - 'Instead got %s.' % labels.dtype) + raise ValueError('Labels dtype should be integer. Instead got {}.'. + format(labels.dtype)) label_ids = labels else: if labels.dtype != dtypes.string: @@ -520,7 +521,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): # Train. if train_op_fn is None: - raise ValueError('train_op_fn can not be None.') + raise ValueError('train_op_fn cannot be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), @@ -555,11 +556,11 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss( generated for each threshold value. This threshold is applied to the logistic values to determine the binary classification (i.e., above the threshold is `true`, below is `false`. - label_vocabulary: A list of strings represents possible label values. If it - is not given, that means labels are already encoded within [0, 1]. If - given, labels must be string type and have any value in - `label_vocabulary`. Also there will be errors if vocabulary is not - provided and labels are string. + label_vocabulary: A list or tuple of strings representing possible label + values. If it is not given, that means labels are already encoded within + [0, 1]. If given, labels must be string type and have any value in + `label_vocabulary`. Note that errors will be raised if `label_vocabulary` + is not provided but labels are strings. name: name of the head. If provided, summary and metrics keys will be suffixed by `"/" + name`. Also used as `name_scope` when creating ops. @@ -572,12 +573,13 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss( thresholds = tuple(thresholds) if thresholds else tuple() if label_vocabulary is not None and not isinstance(label_vocabulary, (list, tuple)): - raise ValueError('label_vocabulary should be a list. Given type: {}'.format( - type(label_vocabulary))) + raise ValueError( + 'label_vocabulary should be a list or tuple. Given type: {}'.format( + type(label_vocabulary))) for threshold in thresholds: if (threshold <= 0.0) or (threshold >= 1.0): - raise ValueError('thresholds not in (0, 1): %s.' % (thresholds,)) + raise ValueError('thresholds not in (0, 1): {}.'.format((thresholds,))) return _BinaryLogisticHeadWithSigmoidCrossEntropyLoss( weight_column=weight_column, thresholds=thresholds, -- GitLab From ed39d70af37b1f794f63e0a77ae14a41271173bf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 15:43:16 -0700 Subject: [PATCH 430/573] Avoid assigning incompatible shapes to variable PiperOrigin-RevId: 173602235 --- tensorflow/python/eager/graph_callable.py | 4 ++-- .../kernel_tests/resource_variable_ops_test.py | 6 ++++-- tensorflow/python/ops/resource_variable_ops.py | 13 ++++++++++++- tensorflow/python/training/saver.py | 5 +++-- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index 7f7a8c4a88..a7f1061d18 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -156,8 +156,8 @@ class _VariableCapturingScope(object): graph_mode_resource = v.variable.handle if initializer is None: initializer = _default_initializer(name, shape, dtype) - resource_variable_ops.assign_variable_op( - graph_mode_resource, initializer(shape, dtype)) + resource_variable_ops.shape_safe_assign_variable_handle( + graph_mode_resource, v.variable.shape, initializer(shape, dtype)) return v.variable scope = variable_scope.get_variable_scope() diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index d8d1ba6bbc..c33bacc5a5 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -305,8 +305,10 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with variable_scope.variable_scope("foo"): var = variable_scope.get_variable("x", shape=[1, 1], dtype=dtypes.float32) - assign = var.assign(np.zeros(shape=[2, 2])) - self.evaluate(assign) + with self.assertRaisesRegexp(ValueError, + "Shapes.*and.*are incompatible"): + assign = var.assign(np.zeros(shape=[2, 2])) + self.evaluate(assign) def testDtypeAfterFromProto(self): v = resource_variable_ops.ResourceVariable(2.0) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index ee8dd08c43..eebb5f217c 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -73,6 +73,15 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): return handle +def shape_safe_assign_variable_handle(handle, shape, value, name=None): + """Helper that checks shape compatibility and assigns variable.""" + value_tensor = ops.convert_to_tensor(value) + shape.assert_is_compatible_with(value_tensor.shape) + return gen_resource_variable_ops.assign_variable_op(handle, + value_tensor, + name=name) + + class ResourceVariable(variables.Variable): """Variable based on resource handles. @@ -755,10 +764,12 @@ class ResourceVariable(variables.Variable): return self.read_value() def assign(self, value, use_locking=None, name=None): + value_tensor = ops.convert_to_tensor(value, dtype=self.dtype) + self._shape.assert_is_compatible_with(value_tensor.shape) with ops.control_dependencies([ gen_resource_variable_ops.assign_variable_op( self.handle, - ops.convert_to_tensor(value, dtype=self.dtype), + value_tensor, name=name) ]): return self.read_value() diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index c4c1df22eb..145b44e2e0 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -164,6 +164,7 @@ class BaseSaverBuilder(object): def __init__(self, var, slice_spec, name): self._var_device = var.device + self._var_shape = var.shape if isinstance(var, ops.Tensor): self.handle_op = var.op.inputs[0] tensor = var @@ -194,8 +195,8 @@ class BaseSaverBuilder(object): # Copy the restored tensor to the variable's device. with ops.device(self._var_device): restored_tensor = array_ops.identity(restored_tensor) - return resource_variable_ops.assign_variable_op( - self.handle_op, restored_tensor) + return resource_variable_ops.shape_safe_assign_variable_handle( + self.handle_op, self._var_shape, restored_tensor) def __init__(self, write_version=saver_pb2.SaverDef.V2): self._write_version = write_version -- GitLab From 2f8d3387c3b27cd77509efb69732c336479e014c Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Thu, 26 Oct 2017 16:02:08 -0700 Subject: [PATCH 431/573] [XLA] Fix race condition in LocalClientTestBase. GetOrCreateAllocator needs to be thread-safe. PiperOrigin-RevId: 173604923 --- tensorflow/compiler/xla/tests/local_client_test_base.cc | 3 +++ tensorflow/compiler/xla/tests/local_client_test_base.h | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index 05e282d208..c11e1df0a7 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -90,6 +90,9 @@ int64 TestAllocator::deallocation_count(int device_ordinal) const { /* static */ TestAllocator* LocalClientTestBase::GetOrCreateAllocator( perftools::gputools::Platform* platform) { + static tensorflow::mutex mu(tensorflow::LINKER_INITIALIZED); + tensorflow::mutex_lock lock(mu); + if (allocator_ == nullptr) { allocator_ = new TestAllocator( platform == nullptr ? PlatformUtil::GetDefaultPlatform().ValueOrDie() diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.h b/tensorflow/compiler/xla/tests/local_client_test_base.h index 17c25adfef..3edfcb656e 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.h +++ b/tensorflow/compiler/xla/tests/local_client_test_base.h @@ -128,8 +128,8 @@ class LocalClientTestBase : public ::testing::Test { return ::testing::UnitTest::GetInstance()->current_test_info()->name(); } - // The allocator must live as long as the service which lives until the end of - // the process, so make the allocator static. + // The allocator must live as long as the service, which lives until the end + // of the process. So make the allocator static. static TestAllocator* allocator_; perftools::gputools::StreamExecutor* stream_executor_; -- GitLab From bcf5dcc87ed2fe05197beaef30e536575608700b Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 26 Oct 2017 16:21:32 -0700 Subject: [PATCH 432/573] eager: Update use of Datasets. 1. Use tf.data instead of tf.contrib.data 2. Move the CPU->device copy to tfe.Iterator. (So that users of tfe.Iterator don't have to do so themselves) The latter is implemented using tf.identity() to explicitly copy each element of the dataset, which can become a bottleneck. This should be replaced by a scheme there the dataset prefetches elements into device memory. PiperOrigin-RevId: 173607457 --- tensorflow/contrib/eager/python/BUILD | 3 ++- tensorflow/contrib/eager/python/datasets.py | 26 ++++++++++++++----- .../contrib/eager/python/datasets_test.py | 10 ++++++- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index adfaaa010a..179c27ba80 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -49,6 +49,7 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ + "//tensorflow/python:array_ops", "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:errors", "//tensorflow/python:resource_variable_ops", @@ -63,8 +64,8 @@ py_test( srcs_version = "PY2AND3", deps = [ ":datasets", - "//tensorflow/contrib/data", "//tensorflow/python:math_ops", + "//tensorflow/python/data", "//tensorflow/python/eager:test", "//third_party/py/numpy", ], diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py index fb9fabd6c1..f83c470411 100644 --- a/tensorflow/contrib/eager/python/datasets.py +++ b/tensorflow/contrib/eager/python/datasets.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Support for tf.contrib.data when eager execution is enabled.""" +"""Iteration over tf.data.Datasets when eager execution is enabled.""" from __future__ import absolute_import from __future__ import division @@ -24,6 +24,7 @@ from tensorflow.python.data.util import nest from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import resource_variable_ops @@ -40,20 +41,23 @@ def _iterator_shared_name(): class Iterator(object): - """An iterator producing tf.Tensor objects from a tf.contrib.data.Dataset.""" + """An iterator producing tf.Tensor objects from a tf.data.Dataset.""" def __init__(self, dataset): """Creates a new iterator over the given dataset. For example: ```python - dataset = tf.contrib.data.Dataset.range(4) + dataset = tf.data.Dataset.range(4) for x in Iterator(dataset): print(x) ``` + Tensors produced will be placed on the device on which this iterator object + was created. + Args: - dataset: A `tf.contrib.data.Dataset` object. + dataset: A `tf.data.Dataset` object. Raises: RuntimeError: When invoked without eager execution enabled. @@ -61,8 +65,10 @@ class Iterator(object): if not context.in_eager_mode(): raise RuntimeError( - "{} objects only make sense when eager execution is enabled".format( - type(self))) + "{} objects can only be used when eager execution is enabled, use " + "tf.data.Dataset.make_iterator or " + "tf.data.Dataset.make_one_shot_iterator for graph construction". + format(type(self))) with ops.device("/device:CPU:0"): ds_variant = dataset._as_variant_tensor() # pylint: disable=protected-access self._output_types = dataset.output_types @@ -74,6 +80,7 @@ class Iterator(object): output_types=self._flat_output_types, output_shapes=self._flat_output_shapes) gen_dataset_ops.make_iterator(ds_variant, self._resource) + self._device = context.context().device_name def __del__(self): if self._resource is not None: @@ -98,6 +105,11 @@ class Iterator(object): self._resource, output_types=self._flat_output_types, output_shapes=self._flat_output_shapes) - return nest.pack_sequence_as(self._output_types, ret) except errors.OutOfRangeError: raise StopIteration + # Copies tensors from CPU to the current device if necessary. + # TODO(rohanj): This should be replaced by the mechanism to have the + # runtime's threads copy tensors to the destination device. + with ops.device(self._device): + ret = [array_ops.identity(x) for x in ret] + return nest.pack_sequence_as(self._output_types, ret) diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py index 076c92e73f..c924d81c9d 100644 --- a/tensorflow/contrib/eager/python/datasets_test.py +++ b/tensorflow/contrib/eager/python/datasets_test.py @@ -16,10 +16,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.data import Dataset from tensorflow.contrib.eager.python import datasets +from tensorflow.python.data import Dataset from tensorflow.python.eager import test from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import script_ops @@ -81,6 +82,13 @@ class IteratorTest(test.TestCase): got = [x.numpy() for x in datasets.Iterator(ds)] self.assertAllEqual([[1], [2], [3], [4]], got) + def testTensorsPlacedOnDevice(self): + ds = Dataset.from_tensors([0., 1.]) + with ops.device(test.gpu_device_name()): + x = datasets.Iterator(ds).next() + x = math_ops.add(x, x) + self.assertAllEqual([0., 2.], x.numpy()) + if __name__ == '__main__': test.main() -- GitLab From 53c4408078d5e4e095a2661e7e6547782391ded8 Mon Sep 17 00:00:00 2001 From: Thomas Schumm Date: Thu, 26 Oct 2017 16:34:28 -0700 Subject: [PATCH 433/573] Let users check for an hparam's existence in a more readable way. PiperOrigin-RevId: 173608993 --- tensorflow/contrib/training/python/training/hparam.py | 3 +++ tensorflow/contrib/training/python/training/hparam_test.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py index 1b52d23c61..391899b34f 100644 --- a/tensorflow/contrib/training/python/training/hparam.py +++ b/tensorflow/contrib/training/python/training/hparam.py @@ -532,6 +532,9 @@ class HParams(object): """ return {n: getattr(self, n) for n in self._hparam_types.keys()} + def __contains__(self, key): + return key in self._hparam_types + def __str__(self): return str(sorted(self.values().items())) diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py index a947bf6eda..f54514cefd 100644 --- a/tensorflow/contrib/training/python/training/hparam_test.py +++ b/tensorflow/contrib/training/python/training/hparam_test.py @@ -32,6 +32,11 @@ class HParamsTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'): hparams.parse('xyz=123') + def testContains(self): + hparams = hparam.HParams(foo=1) + self.assertTrue('foo' in hparams) + self.assertFalse('bar' in hparams) + def testSomeValues(self): hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6') self.assertDictEqual({'aaa': 1, 'b': 2.0, 'c_c': 'relu6'}, hparams.values()) -- GitLab From 8772f8c4eb96a9e1a394fd247ca3c7da8f71d38f Mon Sep 17 00:00:00 2001 From: Russell Power Date: Thu, 26 Oct 2017 16:36:15 -0700 Subject: [PATCH 434/573] Some simple testing utilities for easing unittest writing with TPU devices. PiperOrigin-RevId: 173609207 --- tensorflow/contrib/tpu/BUILD | 12 ++ .../contrib/tpu/python/tpu/test_util.py | 153 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 tensorflow/contrib/tpu/python/tpu/test_util.py diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 970fc97605..c89596734c 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -30,6 +30,18 @@ cc_library( ], ) +py_library( + name = "tpu_test_util", + srcs = [ + "python/tpu/test_util.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":tpu_lib", + ":tpu_py", + ], +) + py_library( name = "tpu_estimator", srcs = [ diff --git a/tensorflow/contrib/tpu/python/tpu/test_util.py b/tensorflow/contrib/tpu/python/tpu/test_util.py new file mode 100644 index 0000000000..f30c27f129 --- /dev/null +++ b/tensorflow/contrib/tpu/python/tpu/test_util.py @@ -0,0 +1,153 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =================================================================== +"""Utilities to ease testing on TPU devices.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.tpu.python.tpu import tpu + +from tensorflow.python.client import session +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import variables + + +def has_tpu(): + """Check if a TPU device is available. + + Device enumeration via `device_lib` currently fails for TPU systems. + (http://b/68333779). To work around this, we determine the existence of a + TPU by a successful call to `initialize_system`. + + Returns: + boolean, True if a TPU device is available, otherwise False. + """ + def _check(): + with session.Session() as sess: + sess.run(tpu.initialize_system()) + sess.run(tpu.shutdown_system()) + + try: + _check() + return True + except errors.OpError as _: + return False + + +def _available_devices(): + devices = ["cpu"] + if not test_util.gpu_device_name(): + devices.append("gpu") + + if has_tpu(): + devices.append("tpu") + + return tuple(devices) + + +class TPUTestCase(test_util.TensorFlowTestCase): + """Adds helpers for testing on TPU devices to `TensorFlowTestCase`. + + Example usage: + + ``` + def model_fn(features): + return tf.reduce_sum(features * 2) + + class ModelTests(test_util.TPUTestCase): + def test_sum(self): + v = np.random.randn(10, 10).astype("float32") + self.assert_device_output(model_fn, [v], (v*2).sum(), + devices=("cpu", "tpu")) + ``` + """ + + def __init__(self, methodName="runTest"): # pylint: disable=invalid-name + super(TPUTestCase, self).__init__(methodName) + self._available_devices = _available_devices() + + def run_on_device(self, model_fn, model_inputs, device): + """Runs `model_fn` on the given device. + + Raises an exception if no such device is available. `model_fn` should + return one or more tensors as a list or tuple. + + Args: + model_fn: Function returning one or more tensors. + model_inputs: An iterable of Numpy arrays or scalars. + These will be passed as arguments to `model_fn`. + device: Device to run on. One of ("tpu", "gpu", "cpu"). + + Returns: + Output from the model function. + """ + def _make_placeholders(): + return dict( + [(gen_array_ops.placeholder_with_default(v, v.shape), v) + for v in model_inputs]) + + if device == "tpu": + with self.test_session(graph=ops.Graph()) as sess: + placeholders = _make_placeholders() + tpu_computation = tpu.rewrite(model_fn, placeholders.keys()) + sess.run(tpu.initialize_system()) + sess.run(variables.global_variables_initializer()) + result = sess.run(tpu_computation, placeholders) + sess.run(tpu.shutdown_system()) + # TODO(b/36891278): supports non-flat returns lists in tpu.rewrite(). + if len(result) == 1: + return result[0] + return result + elif device == "gpu": + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + placeholders = _make_placeholders() + sess.run(variables.global_variables_initializer()) + return sess.run(model_fn(placeholders.keys()), placeholders) + elif device == "cpu": + # TODO(power) -- will this interact poorly with cached GPU sessions? + with self.test_session(graph=ops.Graph(), use_gpu=False) as sess: + placeholders = _make_placeholders() + sess.run(variables.global_variables_initializer()) + return sess.run(model_fn(placeholders.keys()), placeholders) + + def _compare_values(self, actual_outputs, expected_outputs): + if isinstance(expected_outputs, (list, tuple)): + for a, b in zip(actual_outputs, expected_outputs): + self.assertAllCloseAccordingToType(a, b) + else: + self.assertAllCloseAccordingToType(actual_outputs, expected_outputs) + + def assert_device_output(self, model_fn, model_inputs, expected_outputs, + devices=("cpu", "gpu", "tpu")): + """Run `model_fn` on the given devices. + + Results are compared via `assertAllCloseAccordingToType`. + + Args: + model_fn: Function returning one or more tensors + model_inputs: Numpy arrays or scalars passed as arguments to model_fn + expected_outputs: Numpy arrays or scalars to compare against. + devices: Set of devices to run on. If a device is not available, tests + will be skipped for that device. + """ + devices = set(devices).intersection(self._available_devices) + + for device in devices: + device_out = self.run_on_device(model_fn, model_inputs, device=device) + self._compare_values(device_out, expected_outputs) -- GitLab From 19a795bfbe46bfc5557f19f38c7a47242aa8927b Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Thu, 26 Oct 2017 16:38:19 -0700 Subject: [PATCH 435/573] Introduce Eager-specific and Graph-specific Python TensorArray classes, and refactor the Python TensorArray class to wrap an instance of one of these two classes; the particular class to instantiate is chosen by consulting whether the context is in graph or eager mode. TensorArrays in Eager are simply Python lists of EagerTensor objects; most operations on an _EagerTensorArray object pass through to array_ops. This change is meant to ensure compatibility of Eager execution with existing code bases (e.g., code using control_flow_ops) that use TensorArray objects. Eager users will be better served by maintaining their own lists of EagerTensors. PiperOrigin-RevId: 173609453 --- tensorflow/python/BUILD | 3 + tensorflow/python/kernel_tests/BUILD | 5 + .../kernel_tests/tensor_array_ops_test.py | 436 ++++++---- tensorflow/python/ops/tensor_array_ops.py | 769 +++++++++++++++--- 4 files changed, 910 insertions(+), 303 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e2be7e8e9a..4de5d7f7db 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2298,7 +2298,10 @@ py_library( srcs_version = "PY2AND3", deps = [ ":array_ops", + ":constant_op", ":data_flow_ops_gen", + ":dtypes", + ":errors", ":framework_ops", ":math_ops", ":tensor_shape", diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index d8ecabcdea..63844177b7 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2012,13 +2012,18 @@ cuda_py_test( "//tensorflow/python:data_flow_ops_gen", "//tensorflow/python:distributed_framework_test_lib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", + "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn_grad", "//tensorflow/python:training", "//tensorflow/python:tensor_array_grad", "//tensorflow/python:tensor_array_ops", "//tensorflow/python:variables", + "//tensorflow/python:variable_scope", + "//tensorflow/python/eager:backprop", + "//tensorflow/python/eager:context", ], flaky = 1, # create_local_cluster sometimes times out. ) diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py index 53e045fe86..a1fc6d63d4 100644 --- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py +++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py @@ -22,17 +22,22 @@ import numpy as np from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as session_lib +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_data_flow_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_grad from tensorflow.python.ops import tensor_array_ops +from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.platform import test @@ -51,6 +56,11 @@ def _make_converter(tf_dtype): return _converter +def _make_ta(size, name, dtype=dtypes.float32, infer_shape=False): + return tensor_array_ops.TensorArray( + dtype=dtype, tensor_array_name=name, size=size, infer_shape=infer_shape) + + class TensorArrayTest(test.TestCase): @classmethod @@ -63,8 +73,9 @@ class TensorArrayTest(test.TestCase): super(TensorArrayTest, cls).tearDownClass() session_lib.Session.reset(cls._workers[0].target) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteRead(self): - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", @@ -79,7 +90,7 @@ class TensorArrayTest(test.TestCase): r1 = w2.read(1) r2 = w2.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual([[4.0, 5.0]], d0) self.assertAllEqual([[1.0]], d1) self.assertAllEqual(-3.0, d2) @@ -97,8 +108,9 @@ class TensorArrayTest(test.TestCase): c0 = w2.stack() + c0 = self.evaluate(c0) self.assertAllEqual( - convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0.eval()) + convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0) def _testTensorArrayWritePackMaybeLegacy(self): self._testTensorArrayWritePack(dtypes.float32) @@ -109,9 +121,11 @@ class TensorArrayTest(test.TestCase): self._testTensorArrayWritePack(dtypes.complex128) self._testTensorArrayWritePack(dtypes.string) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWritePack(self): self._testTensorArrayWritePackMaybeLegacy() + @test_util.run_in_graph_and_eager_modes() def testEmptyTensorArrayPack(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -124,7 +138,8 @@ class TensorArrayTest(test.TestCase): c0 = w2.stack() - self.assertAllEqual([3, 0, 1], c0.eval().shape) + c0 = self.evaluate(c0) + self.assertAllEqual([3, 0, 1], c0.shape) def _testTensorArrayWriteConcat(self, tf_dtype): with self.test_session(use_gpu=True): @@ -139,10 +154,12 @@ class TensorArrayTest(test.TestCase): c0 = w2.concat() + c0 = self.evaluate(c0) self.assertAllEqual( convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0], - [106.0, 107.0], [8.0, 9.0]]), c0.eval()) + [106.0, 107.0], [8.0, 9.0]]), c0) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteConcat(self): self._testTensorArrayWriteConcat(dtypes.float32) self._testTensorArrayWriteConcat(dtypes.float64) @@ -159,55 +176,46 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError("Could not read from TensorArray index 1 " "because it has not yet been written to."): - ta.write(0, [[4.0, 5.0]]).stack().eval() + self.evaluate(ta.write(0, [[4.0, 5.0]]).stack()) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayPackNotAllValuesAvailableFails(self): self._testTensorArrayPackNotAllValuesAvailableFails() def _testTensorArrayUnpackRead(self, tf_dtype): - with self.test_session(use_gpu=True) as session: - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3) - + with self.test_session(use_gpu=True): convert = _make_converter(tf_dtype) + ta = _make_ta(3, "foo", dtype=tf_dtype) # Unpack a vector into scalars w0 = ta.unstack(convert([1.0, 2.0, 3.0])) r0 = w0.read(0) r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert(1.0), d0) self.assertAllEqual(convert(2.0), d1) self.assertAllEqual(convert(3.0), d2) - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3) - # Unpack a matrix into vectors w1 = ta.unstack(convert([[1.0, 1.1], [2.0, 2.1], [3.0, 3.1]])) r0 = w1.read(0) r1 = w1.read(1) r2 = w1.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([1.0, 1.1]), d0) self.assertAllEqual(convert([2.0, 2.1]), d1) self.assertAllEqual(convert([3.0, 3.1]), d2) - # Reset ta because we're going to change the shape, else shape - # inference will throw an error. - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3) - # Try unpacking an empty matrix, which should not cause an error. w2 = ta.unstack(convert([[], [], []])) r0 = w2.read(0) r1 = w2.read(1) r2 = w2.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([]), d0) self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([]), d2) @@ -221,24 +229,23 @@ class TensorArrayTest(test.TestCase): self._testTensorArrayUnpackRead(dtypes.complex128) self._testTensorArrayUnpackRead(dtypes.string) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayUnpackRead(self): self._testTensorArrayUnpackReadMaybeLegacy() def _testTensorArraySplitRead(self, tf_dtype): - with self.test_session(use_gpu=True) as session: - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3, infer_shape=False) - + with self.test_session(use_gpu=True): convert = _make_converter(tf_dtype) # Split an empty vector + ta = _make_ta(3, "foo", dtype=tf_dtype) lengths = constant_op.constant([0, 0, 0]) w0 = ta.split(convert([]), lengths=lengths) r0 = w0.read(0) r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([]), d0) self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([]), d2) @@ -250,7 +257,7 @@ class TensorArrayTest(test.TestCase): r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([1.0, 2.0]), d0) self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([3.0]), d2) @@ -263,11 +270,12 @@ class TensorArrayTest(test.TestCase): r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([[1.0, 101.0], [2.0, 201.0]]), d0) self.assertAllEqual(convert([]).reshape(0, 2), d1) self.assertAllEqual(convert([[3.0, 301.0]]), d2) + @test_util.run_in_graph_and_eager_modes() def testTensorArraySplitRead(self): self._testTensorArraySplitRead(dtypes.float32) self._testTensorArraySplitRead(dtypes.float64) @@ -367,59 +375,76 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual(t_g_ta_0, t_g_ta_1) self.assertAllEqual([[4.0, 5.0]], d_r1_0) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteWrongIndexOrDataTypeFails(self): with self.test_session(use_gpu=True): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, tensor_array_name="foo", size=3) - + ta = _make_ta(3, "foo", dtype=dtypes.float32) + in_graph_mode = context.in_graph_mode() # Test writing the wrong datatype - with self.assertRaisesOpError( - "TensorArray dtype is float but Op is trying to write dtype string"): - ta.write(-1, "wrong_type_scalar").flow.eval() - - # Test writing to a negative index - with self.assertRaisesOpError( - "Tried to write to index -1 but array is not " - "resizeable and size is: 3"): - ta.write(-1, 3.0).flow.eval() + if in_graph_mode: + with self.assertRaisesOpError( + "TensorArray dtype is float but Op is trying to write " + "dtype string"): + self.evaluate(ta.write(0, "wrong_type_scalar").flow) + else: + with self.assertRaisesOpError( + "TensorArray dtype is float32 but Op is trying to write " + "dtype string"): + self.evaluate(ta.write(0, "wrong_type_scalar").flow) + + if context.in_graph_mode(): + with self.assertRaisesOpError( + "Tried to write to index -1 but array is not " + "resizeable and size is: 3"): + self.evaluate(ta.write(-1, 3.0).flow) + else: + with self.assertRaisesOpError( + r"Writing to negative indices \(index -1\) is not allowed."): + self.evaluate(ta.write(-1, 3.0).flow) # Test reading from too large an index with self.assertRaisesOpError( "Tried to write to index 3 but array is not " "resizeable and size is: 3"): - ta.write(3, 3.0).flow.eval() + self.evaluate(ta.write(3, 3.0).flow) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayReadWrongIndexOrDataTypeFails(self): with self.test_session(use_gpu=True): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, tensor_array_name="foo", size=3) + ta = _make_ta(3, "foo", dtype=dtypes.float32) w0 = ta.write(0, [[4.0, 5.0]]) - # Test reading wrong datatype - r0_bad = gen_data_flow_ops._tensor_array_read_v3( - handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) - with self.assertRaisesOpError( - "TensorArray dtype is float but Op requested dtype double."): - r0_bad.eval() + # Test reading wrong datatype, which is only possible in graph mode + if context.in_graph_mode(): + r0_bad = gen_data_flow_ops._tensor_array_read_v3( + handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) + with self.assertRaisesOpError( + "TensorArray dtype is float but Op requested dtype double."): + r0_bad.eval() # Test reading from a different index than the one we wrote to - r1 = w0.read(1) with self.assertRaisesOpError( "Could not read from TensorArray index 1 because " "it has not yet been written to."): - r1.eval() + self.evaluate(w0.read(1)) - # Test reading from a negative index - with self.assertRaisesOpError( - r"Tried to read from index -1 but array size is: 3"): - ta.read(-1).eval() + # Test reading from a negative index, which is not allowed + if context.in_graph_mode(): + with self.assertRaisesOpError( + r"Tried to read from index -1 but array size is: 3"): + self.evaluate(ta.read(-1)) + else: + with self.assertRaisesOpError( + r"Reading from negative indices \(index -1\) is not allowed."): + self.evaluate(ta.read(-1)) # Test reading from too large an index with self.assertRaisesOpError( "Tried to read from index 3 but array size is: 3"): - ta.read(3).eval() + self.evaluate(ta.read(3)) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteMultipleFails(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -428,8 +453,12 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError( "Could not write to TensorArray index 2 because " "it has already been written to."): - ta.write(2, 3.0).write(2, 3.0).flow.eval() + if context.in_graph_mode(): + self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow) + else: + self.evaluate(ta.write(2, 3.0).write(2, 3.0)) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayConcatIncompatibleShapesFails(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -444,7 +473,7 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError( "Concat saw a scalar shape at index 0 but requires at least vectors"): - w3.concat().eval() + self.evaluate(w3.concat()) ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -456,45 +485,58 @@ class TensorArrayTest(test.TestCase): w2 = w1.write(1, [4.0]) w3 = w2.write(2, [[3.0]]) - with self.assertRaisesOpError( - r"TensorArray has inconsistent shapes. Index 0 has " - r"\(excepting dimension 0\) shape: \[\] but index 2 has \(excepting " - r"dimension 0\) shape: \[1\]"): - w3.concat().eval() + # The eager-mode implementation just passes up array_op.concat's error + # message. + if context.in_graph_mode(): + with self.assertRaisesOpError( + r"TensorArray has inconsistent shapes. Index 0 has " + r"\(excepting dimension 0\) shape: \[\] but index 2 has " + r"\(excepting dimension 0\) shape: \[1\]"): + self.evaluate(w3.concat()) + else: + with self.assertRaisesOpError( + r".*Ranks of all input tensors should match: shape\[0\] " + r"= \[1\] vs\. shape\[2\] = \[1,1\].*"): + self.evaluate(w3.concat()) + @test_util.run_in_graph_and_eager_modes() def testTensorArraySplitIncompatibleShapesFails(self): with self.test_session(use_gpu=True): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, - tensor_array_name="foo", - size=3, - infer_shape=False) - + in_graph_mode = context.in_graph_mode() + ta = _make_ta(3, "foo") with self.assertRaisesOpError( r"Expected lengths to be a vector, received shape: \[\]"): - lengths = array_ops.placeholder(dtypes.int64) - ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1}) + if in_graph_mode: + lengths = array_ops.placeholder(dtypes.int64) + ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1}) + else: + self.evaluate(ta.split([1.0, 2.0, 3.0], 1)) with self.assertRaisesOpError( r"Expected sum of lengths to be equal to values.shape\[0\], " r"but sum of lengths is 1 and value's shape is: \[3\]"): - ta.split([1.0, 2.0, 3.0], [1]).flow.eval() + if in_graph_mode: + self.evaluate(ta.split([1.0, 2.0, 3.0], [1]).flow) + else: + self.evaluate(ta.split([1.0, 2.0, 3.0], [1])) + ta = _make_ta(1, "baz") with self.assertRaisesOpError( r"Expected value to be at least a vector, but received shape: \[\]"): - ta.split(1.0, [1]).flow.eval() - - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, - tensor_array_name="foo", - size=2, - infer_shape=False) + if in_graph_mode: + self.evaluate(ta.split(1.0, [1]).flow) + else: + self.evaluate(ta.split(1.0, [1])) + ta = _make_ta(2, "buz") with self.assertRaisesOpError( r"TensorArray's size is not equal to the size of lengths " r"\(2 vs. 1\), and the TensorArray is not marked as " r"dynamically resizeable"): - ta.split([1.0], [1]).flow.eval() + if in_graph_mode: + self.evaluate(ta.split([1.0], [1]).flow) + else: + self.evaluate(ta.split([1.0], [1])) def _testTensorArrayWriteGradientAddMultipleAdds(self, dtype): with self.test_session(use_gpu=True): @@ -535,6 +577,7 @@ class TensorArrayTest(test.TestCase): dtypes.complex64, dtypes.complex128): self._testTensorArrayWriteGradientAddMultipleAdds(dtype) + @test_util.run_in_graph_and_eager_modes() def testMultiTensorArray(self): with self.test_session(use_gpu=True): h1 = tensor_array_ops.TensorArray( @@ -548,7 +591,8 @@ class TensorArrayTest(test.TestCase): w2 = h2.write(0, 5.0) r2 = w2.read(0) r = r1 + r2 - self.assertAllClose(9.0, r.eval()) + val = self.evaluate(r) + self.assertAllClose(9.0, val) def _testTensorArrayGradientWriteReadType(self, dtype): with self.test_session(use_gpu=True) as session: @@ -637,6 +681,7 @@ class TensorArrayTest(test.TestCase): def testTensorArrayGradientWritePackConcatAndRead(self): self._testTensorArrayGradientWritePackConcatAndRead() + @test_util.run_in_graph_and_eager_modes() def testTensorArrayReadTwice(self): with self.test_session(use_gpu=True): value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) @@ -646,13 +691,12 @@ class TensorArrayTest(test.TestCase): w_readonce = ta_readonce.unstack(value) r0_readonce = w_readonce.read(0) - with ops.control_dependencies([r0_readonce]): - r1_readonce = w_readonce.read(0) with self.assertRaisesOpError( r"Could not read index 0 twice because it was cleared after a " r"previous read \(perhaps try setting clear_after_read = false\?\)"): - r1_readonce.eval() + with ops.control_dependencies([r0_readonce]): + self.evaluate(w_readonce.read(0)) ta_readtwice = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -664,7 +708,7 @@ class TensorArrayTest(test.TestCase): with ops.control_dependencies([r0_readtwice]): r1_readtwice = w_readtwice.read(0) - self.assertAllEqual([1.0, -1.0], r1_readtwice.eval()) + self.assertAllEqual([1.0, -1.0], self.evaluate(r1_readtwice)) def _testTensorArrayGradientUnpackRead(self): with self.test_session(use_gpu=True) as session: @@ -741,20 +785,22 @@ class TensorArrayTest(test.TestCase): def testTensorArrayGradientDynamicUnpackRead(self): self._testTensorArrayGradientDynamicUnpackRead() + @test_util.run_in_graph_and_eager_modes() def testCloseTensorArray(self): - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) - c1 = ta.close() - session.run(c1) + self.evaluate(ta.close()) + @test_util.run_in_graph_and_eager_modes() def testSizeTensorArray(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) s = ta.size() - self.assertAllEqual(3, s.eval()) + self.assertAllEqual(3, self.evaluate(s)) + @test_util.run_in_graph_and_eager_modes() def testWriteCloseTensorArray(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -764,48 +810,62 @@ class TensorArrayTest(test.TestCase): infer_shape=False) w0 = ta.write(0, [[4.0, 5.0]]) w1 = w0.write(1, [3.0]) - w1.close().run() # Expected to run without problems + self.evaluate(w1.close()) # Expected to run without problems def _testWhileLoopWritePackGradients(self, dynamic_size, dtype): np_dtype = dtype.as_numpy_dtype - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): + def func(v0, state0, var): + ta = tensor_array_ops.TensorArray( + dtype=dtype, + tensor_array_name="foo", + size=0 if dynamic_size else 3, + dynamic_size=dynamic_size) + time_0 = array_ops.identity(0) + + def body(time, ta_t, state): + sliced = array_ops.slice( + v0, begin=array_ops.stack([time, 0]), size=[1, -1]) + sliced = array_ops.squeeze(sliced) + out = sliced + var + state + state += sliced + ta_t = ta_t.write(time, out) + return (time + 1, ta_t, state) + + (unused_0, h_final, unused_2) = control_flow_ops.while_loop( + cond=lambda time, unused_1, unused_2: time < 3, + body=body, + loop_vars=(time_0, ta, state0), + shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(), + tensor_shape.unknown_shape()), + parallel_iterations=3) + vout = h_final.stack() + return vout + v0 = array_ops.identity(np.arange(3 * 5, dtype=np_dtype).reshape(3, 5)) - var = variables.Variable(np.arange(100, 105, dtype=np_dtype)) state0 = array_ops.identity(np.array([1] * 5, dtype=np_dtype)) - ta = tensor_array_ops.TensorArray( - dtype=dtype, - tensor_array_name="foo", - size=0 if dynamic_size else 3, - dynamic_size=dynamic_size) - time_0 = array_ops.identity(0) - - def body(time, ta_t, state): - sliced = array_ops.slice( - v0, begin=array_ops.stack([time, 0]), size=[1, -1]) - sliced = array_ops.squeeze(sliced) - out = sliced + var + state - state += sliced - ta_t = ta_t.write(time, out) - return (time + 1, ta_t, state) - - (unused_0, h_final, unused_2) = control_flow_ops.while_loop( - cond=lambda time, unused_1, unused_2: time < 3, - body=body, - loop_vars=(time_0, ta, state0), - shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(), - tensor_shape.unknown_shape()), - parallel_iterations=3) - vout = h_final.stack() - + init_val = np.arange(100, 105, dtype=np_dtype) + var = variable_scope.get_variable( + "var", + shape=init_val.shape, + dtype=np_dtype, + initializer=init_ops.constant_initializer(init_val)) + + vout = func(v0, state0, var) grad_val = -np.arange(3 * 5, dtype=np_dtype).reshape(3, 5) - v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0] - state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0] - var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0] + if context.in_graph_mode(): + v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0] + state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0] + var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0] + variables.global_variables_initializer().run() + else: + grad_fn = backprop.gradients_function(func) + v0_grad, state0_grad, var_grad = grad_fn(v0, state0, var, dy=grad_val) - variables.global_variables_initializer().run() state0_t, var_t, v0_t, vout_t, v0_grad_t, var_grad_t, state0_grad_t = ( - session.run([state0, var, v0, vout, v0_grad, var_grad, state0_grad])) - just_v0_grad_t, = session.run([v0_grad]) + self.evaluate( + ([state0, var, v0, vout, v0_grad, var_grad, state0_grad]))) + just_v0_grad_t = self.evaluate(v0_grad) # state = [ state0 | state0 + v0[0] | state0 + v0[0] + v0[1] ] # vout = [ v0[0] + var + state[0] | @@ -838,6 +898,7 @@ class TensorArrayTest(test.TestCase): self.assertAllClose(grad_val.sum(axis=0), var_grad_t) self.assertAllClose(grad_val.sum(axis=0), state0_grad_t) + @test_util.run_in_graph_and_eager_modes() def testWhileLoopWritePackGradients(self): self._testWhileLoopWritePackGradients( dynamic_size=False, dtype=dtypes.float32) @@ -849,38 +910,45 @@ class TensorArrayTest(test.TestCase): self._testWhileLoopWritePackGradients( dynamic_size=True, dtype=dtypes.float32) + @test_util.run_in_graph_and_eager_modes() def testGradSerialTwoLoops(self): with self.test_session(use_gpu=True): - num_steps = 100 - acc = tensor_array_ops.TensorArray( - dtype=dtypes.float32, - size=num_steps, - clear_after_read=False, - element_shape=tensor_shape.scalar()) - i = constant_op.constant(0, name="i") - x = constant_op.constant(2.0, name="x") + def loop(x): + num_steps = 100 + acc = tensor_array_ops.TensorArray( + dtype=dtypes.float32, + size=num_steps, + clear_after_read=False, + element_shape=tensor_shape.scalar()) + i = constant_op.constant(0, name="i") + + c = lambda i, acc: i < 5 - c = lambda i, acc: i < 5 + def b(i, acc): + x1 = control_flow_ops.cond( + math_ops.equal(i, 0), lambda: x, + lambda: math_ops.multiply(acc.read(i - 1), 2.0)) + return i + 1, acc.write(i, x1) - def b(i, acc): - x1 = control_flow_ops.cond( - math_ops.equal(i, 0), lambda: x, - lambda: math_ops.multiply(acc.read(i - 1), 2.0)) - return i + 1, acc.write(i, x1) + i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc]) - i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc]) + z = constant_op.constant(0.0) - z = constant_op.constant(0.0) + def fn(i, acc): + return i + 1, acc.write(i, z) - def fn(i, acc): - return i + 1, acc.write(i, z) + _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn, + [i1, acc1]) - _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn, - [i1, acc1]) + r = acc2.stack() + return r - r = acc2.stack() - grad = gradients_impl.gradients(r, [x])[0] - self.assertAllClose(31.0, grad.eval()) + x = constant_op.constant(2.0, name="x") + if context.in_graph_mode(): + grad = gradients_impl.gradients(loop(x), [x])[0] + else: + grad = backprop.gradients_function(loop)(x)[0] + self.assertAllClose(31.0, self.evaluate(grad)) def testSumOfTwoReadVariablesWithoutRepeatGrad(self): with self.test_session(use_gpu=True) as session: @@ -1019,6 +1087,7 @@ class TensorArrayTest(test.TestCase): r5 = w5.read(0) self.assertAllEqual([5, 4, 2, 3], r5.get_shape().as_list()) + @test_util.run_in_graph_and_eager_modes() def _testUnpackShape(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1035,8 +1104,15 @@ class TensorArrayTest(test.TestCase): c1 = constant_op.constant([4.0, 5.0]) w1 = w0.write(3, c1) - r1 = w1.read(0) - self.assertAllEqual(c1.get_shape(), r1.get_shape()) + + with self.assertRaisesOpError( + r"Could not read index 0 twice because it was cleared after a " + r"previous read \(perhaps try setting clear_after_read = false\?\)"): + with ops.control_dependencies([r0]): + self.evaluate(w1.read(0)) + + r1 = w1.read(1) + self.assertAllEqual(c1.get_shape(), r1.shape) c2 = constant_op.constant([4.0, 5.0, 6.0]) with self.assertRaises(ValueError): @@ -1045,6 +1121,7 @@ class TensorArrayTest(test.TestCase): def testUnpackShape(self): self._testUnpackShape() + @test_util.run_in_graph_and_eager_modes() def testSplitShape(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1066,10 +1143,14 @@ class TensorArrayTest(test.TestCase): infer_shape=True) w0 = ta1.split(value, [1, 2]) r0 = w0.read(0) - self.assertEqual(r0.get_shape().ndims, None) - self.assertEqual( - tensor_shape.TensorShape( - ta1.handle.op.get_attr("element_shape")).ndims, None) + if context.in_graph_mode(): + self.assertEqual(r0.get_shape().ndims, None) + self.assertEqual( + tensor_shape.TensorShape( + ta1.handle.op.get_attr("element_shape")).ndims, None) + else: + self.assertEqual((1, 2), r0.get_shape()) + self.assertEqual((2, 2), w0.read(1).get_shape()) def testWriteUnknownShape(self): with self.test_session(use_gpu=True): @@ -1137,6 +1218,8 @@ class TensorArrayTest(test.TestCase): def testTensorArrayEvalEmpty(self): self._testTensorArrayEvalEmpty() + # this test is ill-defined for Eager mode --- unpacking an empty tensor + # gives an empty list / there is not equivalent of "mark_used" in Eager def _testTensorArrayEvalEmptyWithDefault(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1180,6 +1263,7 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([10.0, -10.0], read_vals[1]) self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0]) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteGatherAndGradients(self): with self.test_session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( @@ -1188,16 +1272,23 @@ class TensorArrayTest(test.TestCase): size=0, dynamic_size=True) - values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)]) - indices = constant_op.constant([1, 8]) - - w = ta.unstack(values) - g = w.gather(indices) + def func(values): + indices = constant_op.constant([1, 8]) + w = ta.unstack(values) + g = w.gather(indices) + return g + values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)]) + g = func(values) + grad_ys = [[[2.0, 3.0], [4.0, 5.0]]] # Test combined gradients + aggregation of read(0) - grad = gradients_impl.gradients( - ys=[g], xs=[values], grad_ys=[[[2.0, 3.0], [4.0, 5.0]]]) - g_vals, grad_vals = session.run([[g], grad]) + if context.in_graph_mode(): + grad = gradients_impl.gradients(ys=[g], xs=[values], grad_ys=grad_ys) + g_vals, grad_vals = session.run([[g], grad]) + else: + g_vals = [g] + grad_vals = backprop.gradients_function(func)( + values, dy=constant_op.constant(grad_ys[0], dtype=dtypes.float32)) # Gradients for 8 of the 10 unread components are zero. expected_grad = np.zeros((10, 2)) @@ -1316,8 +1407,9 @@ class TensorArrayTest(test.TestCase): self.assertFalse( [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayIdentity(self): - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): ta0 = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2, infer_shape=False) ta1 = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=4, @@ -1326,8 +1418,10 @@ class TensorArrayTest(test.TestCase): ta0 = ta0.write(0, 0.) ta1 = ta1.write(0, 1) - v0 = variables.Variable(0) - v1 = variables.Variable(0) + v0 = variable_scope.get_variable( + "v0", shape=(), initializer=init_ops.zeros_initializer()) + v1 = variable_scope.get_variable( + "v1", shape=(), initializer=init_ops.zeros_initializer()) with ops.control_dependencies([v0.assign_add(1)]): ta0 = ta0.identity() @@ -1344,17 +1438,21 @@ class TensorArrayTest(test.TestCase): # Tests correct properties on new TensorArrays. self.assertEqual(dtypes.float32, ta0.dtype) self.assertEqual(dtypes.int32, ta1.dtype) - self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape()) + if context.in_graph_mode(): + self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape()) + else: + self.assertEqual(tensor_shape.scalar(), read1.get_shape()) self.assertEqual(tensor_shape.scalar(), read1.get_shape()) - variables.global_variables_initializer().run() + if context.in_graph_mode(): + variables.global_variables_initializer().run() - read0_v, read1_v, size0_v, size1_v = session.run( - (read0, read1, size0, size1)) + read0_v, read1_v, size0_v, size1_v = self.evaluate((read0, read1, size0, + size1)) # Tests that the control dependencies was added and executed. - self.assertEqual(1, v0.eval()) - self.assertEqual(1, v1.eval()) + self.assertEqual(1, self.evaluate(v0)) + self.assertEqual(1, self.evaluate(v1)) # Tests correct TensorArray. self.assertEqual(read0_v, 0) diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index 37b4b3bcf9..b4b7ad9d91 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -25,6 +25,9 @@ from __future__ import print_function import contextlib from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util @@ -34,15 +37,11 @@ from tensorflow.python.ops import math_ops from tensorflow.python.util import tf_should_use -# TensorArray object accesses many of the hidden generated ops, but is -# in fact built to wrap these methods. +# _GraphTensorArray accesses many of the hidden generated ops, but is in +# fact built to wrap these methods. # pylint: disable=protected-access -class TensorArray(object): - """Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays. - - This class is meant to be used with dynamic iteration primitives such as - `while_loop` and `map_fn`. It supports gradient back-propagation via special - "flow" control flow dependencies. +class _GraphTensorArray(object): + """Graph-mode implementation of TensorArray. """ def __init__(self, @@ -57,14 +56,7 @@ class TensorArray(object): element_shape=None, colocate_with_first_write_call=True, name=None): - """Construct a new TensorArray or wrap an existing TensorArray handle. - - A note about the parameter `name`: - - The name of the `TensorArray` (even if passed in) is uniquified: each time - a new `TensorArray` is created at runtime it is assigned its own name for - the duration of the run. This avoids name collisions if a `TensorArray` - is created within a `while_loop`. + """Constructs a graph mode TensorArray. Args: dtype: (required) data type of the TensorArray. @@ -79,9 +71,9 @@ class TensorArray(object): This is used when creating the TensorArray handle. If this value is set, handle should be None. handle: (optional) A `Tensor` handle to an existing TensorArray. If this - is set, tensor_array_name should be None. + is set, tensor_array_name should be None. Only supported in graph mode. flow: (optional) A float `Tensor` scalar coming from an existing - `TensorArray.flow`. + `TensorArray.flow`. Only supported in graph mode. infer_shape: (optional, default: True) If True, shape inference is enabled. In this case, all elements must have the same shape. element_shape: (optional, default: None) A `TensorShape` object specifying @@ -170,17 +162,14 @@ class TensorArray(object): @property def flow(self): - """The flow `Tensor` forcing ops leading to this TensorArray state.""" return self._flow @property def dtype(self): - """The data type of this TensorArray.""" return self._dtype @property def handle(self): - """The reference to the TensorArray.""" return self._handle def _merge_element_shape(self, shape): @@ -225,13 +214,7 @@ class TensorArray(object): yield def identity(self): - """Returns a TensorArray with the same content and properties. - - Returns: - A new TensorArray object with flow that ensures the control dependencies - from the contexts will become control dependencies for writes, reads, etc. - Use this object all for subsequent operations. - """ + """See TensorArray.""" flow = array_ops.identity(self._flow) ta = TensorArray( dtype=self._dtype, handle=self._handle, flow=flow, @@ -242,6 +225,7 @@ class TensorArray(object): return ta def grad(self, source, flow=None, name=None): + """See TensorArray.""" # tensor_array_grad requires a flow input when forward # TensorArrays are dynamically sized. This forces the creation # of the grad TensorArray only once the final forward array's size @@ -264,15 +248,7 @@ class TensorArray(object): return g def read(self, index, name=None): - """Read the value at location `index` in the TensorArray. - - Args: - index: 0-D. int32 tensor with the index to read from. - name: A name for the operation (optional). - - Returns: - The tensor at index `index`. - """ + """See TensorArray.""" value = gen_data_flow_ops._tensor_array_read_v3( handle=self._handle, index=index, @@ -285,20 +261,7 @@ class TensorArray(object): @tf_should_use.should_use_result def write(self, index, value, name=None): - """Write `value` into index `index` of the TensorArray. - - Args: - index: 0-D. int32 scalar with the index to write to. - value: N-D. Tensor of type `dtype`. The Tensor to write to this index. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the write occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if there are more writers than specified. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArrayWrite", [self._handle, index, value]): value = ops.convert_to_tensor(value, name="value") if self._infer_shape: @@ -319,35 +282,13 @@ class TensorArray(object): return ta def stack(self, name=None): - """Return the values in the TensorArray as a stacked `Tensor`. - - All of the values must have been written and their shapes must all match. - If input shapes have rank-`R`, then output shape will have rank-`(R+1)`. - - Args: - name: A name for the operation (optional). - - Returns: - All the tensors in the TensorArray stacked into one tensor. - """ + """See TensorArray.""" with ops.colocate_with(self._handle): with ops.name_scope(name, "TensorArrayStack", [self._handle]): return self.gather(math_ops.range(0, self.size()), name=name) def gather(self, indices, name=None): - """Return selected values in the TensorArray as a packed `Tensor`. - - All of selected values must have been written and their shapes - must all match. - - Args: - indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If - the `TensorArray` is not dynamic, `max_value=size()`. - name: A name for the operation (optional). - - Returns: - The in the `TensorArray` selected by `indices`, packed into one tensor. - """ + """See TensorArray.""" if self._element_shape: element_shape = self._element_shape[0] else: @@ -364,17 +305,7 @@ class TensorArray(object): return value def concat(self, name=None): - """Return the values in the TensorArray as a concatenated `Tensor`. - - All of the values must have been written, their ranks must match, and - and their shapes must all match for all dimensions except the first. - - Args: - name: A name for the operation (optional). - - Returns: - All the tensors in the TensorArray concatenated into one tensor. - """ + """See TensorArray.""" if self._element_shape and self._element_shape[0].dims is not None: element_shape_except0 = ( tensor_shape.TensorShape(self._element_shape[0].dims[1:])) @@ -392,22 +323,7 @@ class TensorArray(object): @tf_should_use.should_use_result def unstack(self, value, name=None): - """Unstack the values of a `Tensor` in the TensorArray. - - If input value shapes have rank-`R`, then the output TensorArray will - contain elements whose shapes are rank-`(R-1)`. - - Args: - value: (N+1)-D. Tensor of type `dtype`. The Tensor to unstack. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the unstack occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if the shape inference fails. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArrayUnstack", [self._handle, value]): num_elements = array_ops.shape(value)[0] return self.scatter( @@ -415,21 +331,7 @@ class TensorArray(object): @tf_should_use.should_use_result def scatter(self, indices, value, name=None): - """Scatter the values of a `Tensor` in specific indices of a `TensorArray`. - - Args: - indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If - the `TensorArray` is not dynamic, `max_value=size()`. - value: (N+1)-D. Tensor of type `dtype`. The Tensor to unpack. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the scatter occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if the shape inference fails. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArrayScatter", [self._handle, value, indices]): value = ops.convert_to_tensor(value, name="value") @@ -452,21 +354,7 @@ class TensorArray(object): @tf_should_use.should_use_result def split(self, value, lengths, name=None): - """Split the values of a `Tensor` into the TensorArray. - - Args: - value: (N+1)-D. Tensor of type `dtype`. The Tensor to split. - lengths: 1-D. int32 vector with the lengths to use when splitting - `value` along its first dimension. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the split occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if the shape inference fails. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArraySplit", [self._handle, value, lengths]): value = ops.convert_to_tensor(value, name="value") @@ -494,14 +382,627 @@ class TensorArray(object): return ta def size(self, name=None): - """Return the size of the TensorArray.""" + """See TensorArray.""" return gen_data_flow_ops._tensor_array_size_v3( handle=self._handle, flow_in=self.flow, name=name) @tf_should_use.should_use_result def close(self, name=None): - """Close the current TensorArray.""" + """See TensorArray.""" return gen_data_flow_ops._tensor_array_close_v3( handle=self._handle, name=name) # pylint: enable=protected-access + + +# pylint: disable=protected-access +def _eager_write_no_copy(ta, index, value): + """Writes value into an _EagerTensorArray without creating a new TensorArray. + + Args: + ta: _EagerTensorArray into which to write value. + index: 0-D. int32 scalar with the index to write to. + value: N-D. Tensor of type `dtype`. The Tensor to write to this index. + + Raises: + errors_impl.AlreadyExistsError: attempting to overwrite an entry. + errors_impl.InvalidArgumentError: value dtype does not match `ta`'s dtype. + errors_impl.OutOfRangeError: `index` is out of bounds. + ValueError: shape of `value` is not consistent with inferred shape. + """ + + if isinstance(index, ops.EagerTensor): + index = index.numpy() + + if index < 0: + raise errors_impl.OutOfRangeError( + None, None, + "Writing to negative indices (index %d) is not allowed." % index) + + tensor_array = ta._tensor_array + size = len(tensor_array) + if index >= size: + if not ta._dynamic_size: + raise errors_impl.OutOfRangeError( + None, None, + "Tried to write to index %d but array is not resizeable and size " + "is: %d" % (index, size)) + tensor_array.extend([None for _ in range(index - size + 1)]) + + if not isinstance(value, ops.EagerTensor): + value = constant_op.constant(value) + + if ta._infer_shape: + if ta._element_shape is None: + ta._element_shape = value.shape + elif ta._element_shape != value.shape: + raise ValueError("Incompatible shape for value (%s), expected (%s)" % + (value.shape.as_list(), ta._element_shape.as_list())) + + if ta._dtype != value.dtype: + raise errors_impl.InvalidArgumentError( + None, None, + "TensorArray dtype is %s but Op is trying to write dtype %s" % + (ta._dtype.name, value.dtype.name)) + + if ta._tensor_array[index] is not None: + raise errors_impl.AlreadyExistsError( + None, None, + "Could not write to TensorArray index %d because it has already been " + "written to." % index) + + tensor_array[index] = value + +# pylint: enable=protected-access + + +class _EagerTensorArray(object): + """Eager-mode implementation of TensorArray. + """ + + def __init__(self, + dtype, + size=None, + dynamic_size=None, + clear_after_read=None, + tensor_array_name=None, + handle=None, + flow=None, + infer_shape=True, + element_shape=None, + colocate_with_first_write_call=True, + name=None): + """Constructs an Eager mode TensorArray. + + Args: + dtype: (required) data type of the TensorArray. + size: (optional) int32 scalar `Tensor`: the size of the TensorArray. + Required if handle is not provided. + dynamic_size: (optional) Python bool: If true, writes to the TensorArray + can grow the TensorArray past its initial size. Default: False. + clear_after_read: Boolean (optional, default: True). If True, clear + TensorArray values after reading them. This disables read-many + semantics, but allows early release of memory. + tensor_array_name: unused. + handle: unsupported. + flow: unsupported. + infer_shape: used for error checking, same semantics as TensorArray. + element_shape: used for error checking, same semantics as TensorArray. + colocate_with_first_write_call: unsupported. + name: unsupported. + + Raises: + ValueError: handle or flow are supplied, or if size is not supplied. + """ + + del (flow, tensor_array_name, name) # not meaningful in Eager + + if handle is not None: + raise ValueError("TensorArray handles are not supported in Eager mode.") + if size is None: + raise ValueError("Size must be declared for TensorArrays in Eager mode.") + + # These attributes are not meaningful in Eager, but some library functions + # (e.g., those in control_flow_ops.py) access them to create new tensor + # arrays; as such, we define them for the sake of compatibility. + self._handle = None + # we assign a dummy value to _flow in case other code assumes it to be + # a Tensor + self._flow = constant_op.constant(0, dtype=dtypes.int32) + self._infer_shape = infer_shape + self._element_shape = element_shape + self._colocate_with_first_write_call = colocate_with_first_write_call + + self._dtype = dtype + self._dynamic_size = dynamic_size or False + self._clear_after_read = ( + True if clear_after_read is None else clear_after_read) + self._previously_read_indices = [] + + if isinstance(size, ops.EagerTensor): + size = size.numpy() + self._tensor_array = [None for _ in range(size)] + + @property + def flow(self): + """Flows are not meaningful in Eager; this exists for compatibility.""" + return self._flow + + @property + def dtype(self): + return self._dtype + + @property + def handle(self): + """Handles are not meaningful in Eager; this exists for compatibility.""" + return self._handle + + def _identity_without_array(self): + """Returns a new TensorArray with the same properties as this Eager one. + + NB: Does not set the underlying _tensor_array attribute. + """ + ta = TensorArray( + dtype=self._dtype, + size=len(self._tensor_array), + dynamic_size=self._dynamic_size, + clear_after_read=self._clear_after_read, + handle=self._handle, + flow=self._flow, + infer_shape=self._infer_shape, + element_shape=self._element_shape, + colocate_with_first_write_call=self._colocate_with_first_write_call) + ta._implementation._previously_read_indices = self._previously_read_indices # pylint: disable=protected-access + return ta + + def identity(self): + """See TensorArray.""" + ta = self._identity_without_array() + ta._implementation._tensor_array = [t for t in self._tensor_array] # pylint: disable=protected-access + return ta + + def grad(self, source, flow=None, name=None): + raise NotImplementedError( + "TensorArray.grad is not supported in Eager mode; Eager's gradient " + "implementation does not use/need this function to compute gradients " + "of operations that use TensorArrays.") + + def read(self, index, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + + if isinstance(index, ops.EagerTensor): + index = index.numpy() + + if index < 0: + raise errors_impl.OutOfRangeError( + None, None, + "Reading from negative indices (index %d) is not allowed." % index) + + if index >= len(self._tensor_array): + raise errors_impl.OutOfRangeError( + None, None, "Tried to read from index %d but array size is: %d" % + (index, len(self._tensor_array))) + + tensor = self._tensor_array[index] + if tensor is None: + if index in self._previously_read_indices: + raise errors_impl.InvalidArgumentError( + None, None, + "Could not read index %d twice because it was cleared after " + "a previous read (perhaps try setting clear_after_read = false?)" % + index) + else: + raise errors_impl.InvalidArgumentError( + None, None, + "Could not read from TensorArray index %d because it has not yet " + "been written to." % index) + + if self._clear_after_read: + self._tensor_array[index] = None + self._previously_read_indices.append(index) + return tensor + + def write(self, index, value, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + ta = self.identity() + _eager_write_no_copy(ta._implementation, index, value) # pylint: disable=protected-access + return ta + + def stack(self, name=None): + """See TensorArray.""" + try: + return array_ops.stack(self._tensor_array, name=name) + except ValueError: + if None in self._tensor_array: + idx = self._tensor_array.index(None) + raise errors_impl.InvalidArgumentError( + None, None, "Could not read from TensorArray index %d because " + "it has not yet been written to." % idx) + else: + raise + + def gather(self, indices, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + return array_ops.stack([self._tensor_array[i] for i in indices.numpy()]) + + def concat(self, name=None): + """See TensorArray.""" + try: + return array_ops.concat(self._tensor_array, 0, name=name) + except errors_impl.OpError: + # Reproduce a subset of the error-handling for graph-mode TensorArrays. + shapes = [t.shape for t in self._tensor_array] + ndims = [s.ndims for s in shapes] + if None in self._tensor_array: + # Concatenating empty TensorArrays is permitted if the element + # shape is defined; the output is a tensor with shape + # [0] + self._element_shape[1:] + if all(t is None for t in self._tensor_array): + if self._element_shape is not None: + return constant_op.constant([], shape=[0] + self._element_shape[1:]) + else: + raise errors_impl.UnimplementedError( + None, None, "TensorArray has size zero, but " + "element_shape_except0 %s is not fully defined. Currently only " + "static shapes are supported when concatenating zero-size " + "TensorArrays." % self._element_shape[1:]) + # Concatenating a TensorArray in which some but not all entries have + # been written to is not allowed. + idx = self._tensor_array.index(None) + raise errors_impl.InvalidArgumentError( + None, None, "Could not read from TensorArray index %d because " + "it has not yet been written to." % idx) + elif 0 in ndims: + idx = ndims.index(0) + raise errors_impl.InvalidArgumentError( + None, None, "Concat saw a scalar shape at index %d but requires " + "at least vectors." % idx) + else: + raise + + def unstack(self, value, name=None): + """See TensorArray.""" + tensors = array_ops.unstack(value, name=name) + if len(tensors) > len(self._tensor_array) and not self._dynamic_size: + raise ValueError( + "Cannot unstack %d tensors into a TensorArray of static size %d" % + (len(tensors), len(self._tensors))) + ta = self._identity_without_array() + ta._implementation._tensor_array = tensors # pylint: disable=protected-access + return ta + + def scatter(self, indices, value, name=None): + """See TensorArray.""" + del name # unused in Eager + ta = self.identity() + for index, val in zip(indices.numpy(), array_ops.unstack(value)): + _eager_write_no_copy(ta._implementation, index, val) # pylint: disable=protected-access + return ta + + def split(self, value, lengths, name=None): + """See TensorArray.""" + # error checking to match graph-mode errors + value = constant_op.constant(value) + lengths = constant_op.constant(lengths) + sum_lengths = math_ops.reduce_sum(lengths) + if lengths.shape.ndims != 1: + raise errors_impl.InvalidArgumentError( + None, None, "Expected lengths to be a vector, received shape: %s" % + lengths.shape.as_list()) + elif value.shape.ndims == 0: + raise errors_impl.InvalidArgumentError( + None, None, "Expected value to be at least a vector, " + "but received shape: %s" % value.shape.as_list()) + elif sum_lengths.numpy() != value.shape.as_list()[0]: + raise errors_impl.InvalidArgumentError( + None, None, "Expected sum of lengths to be equal to " + "values.shape[0], but sum of lengths is %d and " + "value's shape is: %s " % (sum_lengths.numpy(), + value.shape.as_list())) + elif not self._dynamic_size and lengths.shape[0] != len(self._tensor_array): + raise errors_impl.InvalidArgumentError( + None, None, "TensorArray's size is not equal to the size of " + "lengths (%d vs. %d), and the TensorArray is not marked as " + "dynamically resizeable" % (len(self._tensor_array), + lengths.shape[0])) + else: + ta = self._identity_without_array() + tensor_array = array_ops.split(value, lengths, name=name) + ta._implementation._tensor_array = tensor_array # pylint: disable=protected-access + return ta + + def size(self, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + return constant_op.constant(len(self._tensor_array)) + + def close(self, name=None): + del name # not meaningful in Eager mode + del self._tensor_array[:] + return + + +# TensorArray is designed to hide an underlying implementation object +# and as such accesses many of that object's hidden fields. +# pylint: disable=protected-access +class TensorArray(object): + """Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays. + + This class is meant to be used with dynamic iteration primitives such as + `while_loop` and `map_fn`. It supports gradient back-propagation via special + "flow" control flow dependencies. + """ + + def __init__(self, + dtype, + size=None, + dynamic_size=None, + clear_after_read=None, + tensor_array_name=None, + handle=None, + flow=None, + infer_shape=True, + element_shape=None, + colocate_with_first_write_call=True, + name=None): + """Construct a new TensorArray or wrap an existing TensorArray handle. + + A note about the parameter `name`: + + The name of the `TensorArray` (even if passed in) is uniquified: each time + a new `TensorArray` is created at runtime it is assigned its own name for + the duration of the run. This avoids name collisions if a `TensorArray` + is created within a `while_loop`. + + Args: + dtype: (required) data type of the TensorArray. + size: (optional) int32 scalar `Tensor`: the size of the TensorArray. + Required if handle is not provided. + dynamic_size: (optional) Python bool: If true, writes to the TensorArray + can grow the TensorArray past its initial size. Default: False. + clear_after_read: Boolean (optional, default: True). If True, clear + TensorArray values after reading them. This disables read-many + semantics, but allows early release of memory. + tensor_array_name: (optional) Python string: the name of the TensorArray. + This is used when creating the TensorArray handle. If this value is + set, handle should be None. + handle: (optional) A `Tensor` handle to an existing TensorArray. If this + is set, tensor_array_name should be None. Only supported in graph mode. + flow: (optional) A float `Tensor` scalar coming from an existing + `TensorArray.flow`. Only supported in graph mode. + infer_shape: (optional, default: True) If True, shape inference + is enabled. In this case, all elements must have the same shape. + element_shape: (optional, default: None) A `TensorShape` object specifying + the shape constraints of each of the elements of the TensorArray. + Need not be fully defined. + colocate_with_first_write_call: If `True`, the TensorArray will be + colocated on the same device as the Tensor used on its first write + (write operations include `write`, `unstack`, and `split`). If `False`, + the TensorArray will be placed on the device determined by the + device context available during its initialization. + name: A name for the operation (optional). + + Raises: + ValueError: if both handle and tensor_array_name are provided. + TypeError: if handle is provided but is not a Tensor. + """ + if context.in_graph_mode(): + implementation = _GraphTensorArray + else: + implementation = _EagerTensorArray + + self._implementation = implementation( + dtype, + size=size, + dynamic_size=dynamic_size, + clear_after_read=clear_after_read, + tensor_array_name=tensor_array_name, + handle=handle, + flow=flow, + infer_shape=infer_shape, + element_shape=element_shape, + colocate_with_first_write_call=colocate_with_first_write_call, + name=name) + + @property + def flow(self): + """The flow `Tensor` forcing ops leading to this TensorArray state.""" + return self._implementation._flow + + @property + def dtype(self): + """The data type of this TensorArray.""" + return self._implementation._dtype + + @property + def handle(self): + """The reference to the TensorArray.""" + return self._implementation._handle + + @property + def _infer_shape(self): + return self._implementation._infer_shape + + @_infer_shape.setter + def _infer_shape(self, infer_shape): + self._implementation._infer_shape = infer_shape + + @property + def _element_shape(self): + return self._implementation._element_shape + + @_element_shape.setter + def _element_shape(self, element_shape): + self._implementation._element_shape = element_shape + + @property + def _colocate_with_first_write_call(self): + return self._implementation._colocate_with_first_write_call + + @property + def _colocate_with(self): + return self._implementation._colocate_with + + @_colocate_with.setter + def _colocate_with(self, colocate_with): + self._implementation._colocate_with = colocate_with + + def identity(self): + """Returns a TensorArray with the same content and properties. + + Returns: + A new TensorArray object with flow that ensures the control dependencies + from the contexts will become control dependencies for writes, reads, etc. + Use this object all for subsequent operations. + """ + return self._implementation.identity() + + def grad(self, source, flow=None, name=None): + return self._implementation.grad(source, flow=flow, name=name) + + def read(self, index, name=None): + """Read the value at location `index` in the TensorArray. + + Args: + index: 0-D. int32 tensor with the index to read from. + name: A name for the operation (optional). + + Returns: + The tensor at index `index`. + """ + return self._implementation.read(index, name=name) + + @tf_should_use.should_use_result + def write(self, index, value, name=None): + """Write `value` into index `index` of the TensorArray. + + Args: + index: 0-D. int32 scalar with the index to write to. + value: N-D. Tensor of type `dtype`. The Tensor to write to this index. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the write occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if there are more writers than specified. + """ + return self._implementation.write(index, value, name=name) + + def stack(self, name=None): + """Return the values in the TensorArray as a stacked `Tensor`. + + All of the values must have been written and their shapes must all match. + If input shapes have rank-`R`, then output shape will have rank-`(R+1)`. + + Args: + name: A name for the operation (optional). + + Returns: + All the tensors in the TensorArray stacked into one tensor. + """ + return self._implementation.stack(name=name) + + def gather(self, indices, name=None): + """Return selected values in the TensorArray as a packed `Tensor`. + + All of selected values must have been written and their shapes + must all match. + + Args: + indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If + the `TensorArray` is not dynamic, `max_value=size()`. + name: A name for the operation (optional). + + Returns: + The tensors in the `TensorArray` selected by `indices`, packed into one + tensor. + """ + return self._implementation.gather(indices, name=name) + + def concat(self, name=None): + """Return the values in the TensorArray as a concatenated `Tensor`. + + All of the values must have been written, their ranks must match, and + and their shapes must all match for all dimensions except the first. + + Args: + name: A name for the operation (optional). + + Returns: + All the tensors in the TensorArray concatenated into one tensor. + """ + return self._implementation.concat(name=name) + + @tf_should_use.should_use_result + def unstack(self, value, name=None): + """Unstack the values of a `Tensor` in the TensorArray. + + If input value shapes have rank-`R`, then the output TensorArray will + contain elements whose shapes are rank-`(R-1)`. + + Args: + value: (N+1)-D. Tensor of type `dtype`. The Tensor to unstack. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the unstack occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if the shape inference fails. + """ + return self._implementation.unstack(value, name=name) + + @tf_should_use.should_use_result + def scatter(self, indices, value, name=None): + """Scatter the values of a `Tensor` in specific indices of a `TensorArray`. + + Args: + indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If + the `TensorArray` is not dynamic, `max_value=size()`. + value: (N+1)-D. Tensor of type `dtype`. The Tensor to unpack. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the scatter occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if the shape inference fails. + """ + return self._implementation.scatter(indices, value, name=name) + + @tf_should_use.should_use_result + def split(self, value, lengths, name=None): + """Split the values of a `Tensor` into the TensorArray. + + Args: + value: (N+1)-D. Tensor of type `dtype`. The Tensor to split. + lengths: 1-D. int32 vector with the lengths to use when splitting + `value` along its first dimension. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the split occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if the shape inference fails. + """ + return self._implementation.split(value, lengths, name=name) + + def size(self, name=None): + """Return the size of the TensorArray.""" + return self._implementation.size(name=name) + + @tf_should_use.should_use_result + def close(self, name=None): + """Close the current TensorArray.""" + return self._implementation.close(name=name) + +# pylint: enable=protected-access -- GitLab From 06a79f5af7c861e695cfc20b7778519950aac9ba Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 17:00:14 -0700 Subject: [PATCH 436/573] Move EyeFunctor to a separate file, and change it to a more efficient implementation (similar to matrix_set_diag). PiperOrigin-RevId: 173611865 --- tensorflow/core/kernels/BUILD | 22 +++++++--- tensorflow/core/kernels/cuda_solvers.h | 14 ------- tensorflow/core/kernels/eye_functor.h | 32 +++++++++++++++ ...olvers_gpu.cu.cc => eye_functor_gpu.cu.cc} | 41 ++++++++----------- tensorflow/core/kernels/matrix_inverse_op.cc | 1 + tensorflow/core/kernels/qr_op_impl.h | 1 + 6 files changed, 67 insertions(+), 44 deletions(-) create mode 100644 tensorflow/core/kernels/eye_functor.h rename tensorflow/core/kernels/{cuda_solvers_gpu.cu.cc => eye_functor_gpu.cu.cc} (62%) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index a3452f2f8c..0274f87ec6 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1691,6 +1691,21 @@ tf_cc_tests( ], ) +tf_kernel_library( + name = "eye_functor", + hdrs = ["eye_functor.h"], + gpu_srcs = [ + "eye_functor_gpu.cu.cc", + "eye_functor.h", + ], + visibility = [":friends"], + deps = [ + "//tensorflow/core:framework", + "//third_party/eigen3", + ], + alwayslink = 0, +) + cc_library( name = "fifo_queue", srcs = ["fifo_queue.cc"], @@ -2255,10 +2270,6 @@ tf_kernel_library( name = "cuda_solvers", srcs = ["cuda_solvers.cc"], hdrs = ["cuda_solvers.h"], - gpu_srcs = [ - "cuda_solvers.h", - "cuda_solvers_gpu.cu.cc", - ], # @local_config_cuda//cuda:cusolver, //third_party/eigen3:blas, # and //third_party/libf2c all contain various parts of BLAS, LAPACK, # and f2c helper functions in global namespace. Tell the compiler to @@ -2328,7 +2339,7 @@ tf_kernel_library( tf_kernel_library( name = "matrix_inverse_op", prefix = "matrix_inverse_op", - deps = LINALG_DEPS, + deps = LINALG_DEPS + if_cuda([":eye_functor"]), ) tf_kernel_library( @@ -2356,6 +2367,7 @@ tf_kernel_library( prefix = "qr_op", deps = LINALG_DEPS + if_cuda([ ":cwise_op", + ":eye_functor", ":matrix_band_part_op", ]), ) diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h index af27eb6c47..3c389a82ab 100644 --- a/tensorflow/core/kernels/cuda_solvers.h +++ b/tensorflow/core/kernels/cuda_solvers.h @@ -27,7 +27,6 @@ limitations under the License. #include "cuda/include/cusolverDn.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/stream_executor.h" @@ -408,19 +407,6 @@ class DeviceLapackInfo : public ScratchSpace { } }; -namespace functor { - -// Helper functor to set a batch of matrices to the identity. -// TODO(rmlarsen): Use this kernel to replace the horribly inefficient tf.eye -// op. -template -struct EyeFunctor { - void operator()(const Device& device, - typename TTypes::Tensor matrix_batch); -}; - -} // namespace functor - template ScratchSpace CudaSolver::GetScratchSpace(const TensorShape& shape, const string& debug_info, diff --git a/tensorflow/core/kernels/eye_functor.h b/tensorflow/core/kernels/eye_functor.h new file mode 100644 index 0000000000..70f093f813 --- /dev/null +++ b/tensorflow/core/kernels/eye_functor.h @@ -0,0 +1,32 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ +#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ + +#include "tensorflow/core/framework/tensor_types.h" + +namespace tensorflow { +namespace functor { + +template +struct EyeFunctor { + void operator()(const Device& device, + typename TTypes::Tensor matrix_batch); +}; + +} // namespace functor +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/eye_functor_gpu.cu.cc similarity index 62% rename from tensorflow/core/kernels/cuda_solvers_gpu.cu.cc rename to tensorflow/core/kernels/eye_functor_gpu.cu.cc index 84330c041a..a620316e27 100644 --- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc +++ b/tensorflow/core/kernels/eye_functor_gpu.cu.cc @@ -17,11 +17,11 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/kernels/eye_functor.h" -#include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/cuda_kernel_helper.h" namespace tensorflow { @@ -30,26 +30,18 @@ namespace functor { typedef Eigen::GpuDevice GPUDevice; template -__global__ void EyeKernel(Cuda3DLaunchConfig config, int batch_size, int m, - int n, Scalar* matrix_batch_ptr) { - const int matrix_size = m * n; +__global__ void EyeKernel(int num_threads, int batch_size, int m, int n, + Scalar* output_ptr) { const Scalar one = Scalar(1); - CUDA_AXIS_KERNEL_LOOP(batch, config.virtual_thread_count, x) { - if (batch >= batch_size) { - break; - } - CUDA_AXIS_KERNEL_LOOP(row, config.virtual_thread_count, y) { - if (row >= m) { - break; - } - const int row_start = batch * matrix_size + row * n; - CUDA_AXIS_KERNEL_LOOP(col, config.virtual_thread_count, z) { - if (col >= n) { - break; - } - matrix_batch_ptr[row_start + col] = row == col ? one : Scalar(); - } - } + const Scalar zero = Scalar(0); + CUDA_1D_KERNEL_LOOP(index, num_threads) { + // TODO(rmlarsen): Benchmark to see if it's just as fast to use mod (%), + // since it's easier to read. + const int global_row = index / n; + const int col = index - global_row * n; + const int batch = global_row / m; + const int row = global_row - batch * m; + output_ptr[index] = col == row ? one : zero; } } @@ -60,11 +52,10 @@ struct EyeFunctor { const int batch_size = matrix_batch.dimension(0); const int m = matrix_batch.dimension(1); const int n = matrix_batch.dimension(2); - Cuda3DLaunchConfig config = GetCuda3DLaunchConfig(batch_size, m, n, device, - EyeKernel, 0, 0); + CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * m * n, device); EyeKernel<<>>(config, batch_size, m, n, - matrix_batch.data()); + device.stream()>>>(config.virtual_thread_count, batch_size, m, + n, matrix_batch.data()); } }; diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc index cae84f52d7..c61a091c7b 100644 --- a/tensorflow/core/kernels/matrix_inverse_op.cc +++ b/tensorflow/core/kernels/matrix_inverse_op.cc @@ -33,6 +33,7 @@ limitations under the License. #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/kernels/eye_functor.h" #include "tensorflow/core/kernels/transpose_functor.h" #endif diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/qr_op_impl.h index c51d601437..0552c034d2 100644 --- a/tensorflow/core/kernels/qr_op_impl.h +++ b/tensorflow/core/kernels/qr_op_impl.h @@ -40,6 +40,7 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/cwise_ops.h" +#include "tensorflow/core/kernels/eye_functor.h" #include "tensorflow/core/kernels/matrix_band_part_op.h" #include "tensorflow/core/kernels/transpose_functor.h" #endif -- GitLab From 76c921c42587c6e6f5ece90d0682f0912c5ed3bd Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Thu, 26 Oct 2017 17:05:10 -0700 Subject: [PATCH 437/573] Add a build method to LSTMCell. To make some legacy behavior of LSTMCell work, add the "partitioner" argument to Layer's add_variable method. PiperOrigin-RevId: 173612542 --- .../python/kernel_tests/core_rnn_cell_test.py | 1 - .../rnn/python/kernel_tests/core_rnn_test.py | 150 ++++++------------ tensorflow/python/layers/base.py | 18 ++- tensorflow/python/ops/rnn_cell_impl.py | 108 +++++++------ .../tensorflow.keras.layers.-activation.pbtxt | 2 +- ...eras.layers.-activity-regularization.pbtxt | 2 +- .../golden/tensorflow.keras.layers.-add.pbtxt | 2 +- ...nsorflow.keras.layers.-alpha-dropout.pbtxt | 2 +- ...low.keras.layers.-average-pooling1-d.pbtxt | 2 +- ...low.keras.layers.-average-pooling2-d.pbtxt | 2 +- ...low.keras.layers.-average-pooling3-d.pbtxt | 2 +- .../tensorflow.keras.layers.-average.pbtxt | 2 +- ...tensorflow.keras.layers.-avg-pool1-d.pbtxt | 2 +- ...tensorflow.keras.layers.-avg-pool2-d.pbtxt | 2 +- ...tensorflow.keras.layers.-avg-pool3-d.pbtxt | 2 +- ...ow.keras.layers.-batch-normalization.pbtxt | 2 +- ...nsorflow.keras.layers.-bidirectional.pbtxt | 2 +- ...tensorflow.keras.layers.-concatenate.pbtxt | 2 +- ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt | 2 +- .../tensorflow.keras.layers.-conv1-d.pbtxt | 2 +- ...flow.keras.layers.-conv2-d-transpose.pbtxt | 2 +- .../tensorflow.keras.layers.-conv2-d.pbtxt | 2 +- ...flow.keras.layers.-conv3-d-transpose.pbtxt | 2 +- .../tensorflow.keras.layers.-conv3-d.pbtxt | 2 +- ...sorflow.keras.layers.-convolution1-d.pbtxt | 2 +- ...ras.layers.-convolution2-d-transpose.pbtxt | 2 +- ...sorflow.keras.layers.-convolution2-d.pbtxt | 2 +- ...ras.layers.-convolution3-d-transpose.pbtxt | 2 +- ...sorflow.keras.layers.-convolution3-d.pbtxt | 2 +- ...tensorflow.keras.layers.-cropping1-d.pbtxt | 2 +- ...tensorflow.keras.layers.-cropping2-d.pbtxt | 2 +- ...tensorflow.keras.layers.-cropping3-d.pbtxt | 2 +- .../tensorflow.keras.layers.-dense.pbtxt | 2 +- .../golden/tensorflow.keras.layers.-dot.pbtxt | 2 +- .../tensorflow.keras.layers.-dropout.pbtxt | 2 +- .../tensorflow.keras.layers.-e-l-u.pbtxt | 2 +- .../tensorflow.keras.layers.-embedding.pbtxt | 2 +- .../tensorflow.keras.layers.-flatten.pbtxt | 2 +- .../tensorflow.keras.layers.-g-r-u.pbtxt | 2 +- ...rflow.keras.layers.-gaussian-dropout.pbtxt | 2 +- ...sorflow.keras.layers.-gaussian-noise.pbtxt | 2 +- ...as.layers.-global-average-pooling1-d.pbtxt | 2 +- ...as.layers.-global-average-pooling2-d.pbtxt | 2 +- ...as.layers.-global-average-pooling3-d.pbtxt | 2 +- ...low.keras.layers.-global-avg-pool1-d.pbtxt | 2 +- ...low.keras.layers.-global-avg-pool2-d.pbtxt | 2 +- ...low.keras.layers.-global-avg-pool3-d.pbtxt | 2 +- ...low.keras.layers.-global-max-pool1-d.pbtxt | 2 +- ...low.keras.layers.-global-max-pool2-d.pbtxt | 2 +- ...low.keras.layers.-global-max-pool3-d.pbtxt | 2 +- ....keras.layers.-global-max-pooling1-d.pbtxt | 2 +- ....keras.layers.-global-max-pooling2-d.pbtxt | 2 +- ....keras.layers.-global-max-pooling3-d.pbtxt | 2 +- ...tensorflow.keras.layers.-input-layer.pbtxt | 2 +- .../tensorflow.keras.layers.-l-s-t-m.pbtxt | 2 +- .../tensorflow.keras.layers.-lambda.pbtxt | 2 +- .../tensorflow.keras.layers.-layer.pbtxt | 2 +- ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt | 2 +- ...w.keras.layers.-locally-connected1-d.pbtxt | 2 +- ...w.keras.layers.-locally-connected2-d.pbtxt | 2 +- .../tensorflow.keras.layers.-masking.pbtxt | 2 +- ...tensorflow.keras.layers.-max-pool1-d.pbtxt | 2 +- ...tensorflow.keras.layers.-max-pool2-d.pbtxt | 2 +- ...tensorflow.keras.layers.-max-pool3-d.pbtxt | 2 +- ...sorflow.keras.layers.-max-pooling1-d.pbtxt | 2 +- ...sorflow.keras.layers.-max-pooling2-d.pbtxt | 2 +- ...sorflow.keras.layers.-max-pooling3-d.pbtxt | 2 +- .../tensorflow.keras.layers.-maximum.pbtxt | 2 +- .../tensorflow.keras.layers.-multiply.pbtxt | 2 +- .../tensorflow.keras.layers.-p-re-l-u.pbtxt | 2 +- .../tensorflow.keras.layers.-permute.pbtxt | 2 +- ...nsorflow.keras.layers.-repeat-vector.pbtxt | 2 +- .../tensorflow.keras.layers.-reshape.pbtxt | 2 +- ...flow.keras.layers.-separable-conv2-d.pbtxt | 2 +- ...ras.layers.-separable-convolution2-d.pbtxt | 2 +- ...ensorflow.keras.layers.-simple-r-n-n.pbtxt | 2 +- ...low.keras.layers.-spatial-dropout1-d.pbtxt | 2 +- ...low.keras.layers.-spatial-dropout2-d.pbtxt | 2 +- ...low.keras.layers.-spatial-dropout3-d.pbtxt | 2 +- ...low.keras.layers.-thresholded-re-l-u.pbtxt | 2 +- ...rflow.keras.layers.-time-distributed.pbtxt | 2 +- ...sorflow.keras.layers.-up-sampling1-d.pbtxt | 2 +- ...sorflow.keras.layers.-up-sampling2-d.pbtxt | 2 +- ...sorflow.keras.layers.-up-sampling3-d.pbtxt | 2 +- .../tensorflow.keras.layers.-wrapper.pbtxt | 2 +- ...orflow.keras.layers.-zero-padding1-d.pbtxt | 2 +- ...orflow.keras.layers.-zero-padding2-d.pbtxt | 2 +- ...orflow.keras.layers.-zero-padding3-d.pbtxt | 2 +- .../tensorflow.keras.models.-model.pbtxt | 2 +- .../tensorflow.keras.models.-sequential.pbtxt | 2 +- ...ensorflow.layers.-average-pooling1-d.pbtxt | 2 +- ...ensorflow.layers.-average-pooling2-d.pbtxt | 2 +- ...ensorflow.layers.-average-pooling3-d.pbtxt | 2 +- ...nsorflow.layers.-batch-normalization.pbtxt | 2 +- .../golden/tensorflow.layers.-conv1-d.pbtxt | 2 +- ...tensorflow.layers.-conv2-d-transpose.pbtxt | 2 +- .../golden/tensorflow.layers.-conv2-d.pbtxt | 2 +- ...tensorflow.layers.-conv3-d-transpose.pbtxt | 2 +- .../golden/tensorflow.layers.-conv3-d.pbtxt | 2 +- .../api/golden/tensorflow.layers.-dense.pbtxt | 2 +- .../golden/tensorflow.layers.-dropout.pbtxt | 2 +- .../golden/tensorflow.layers.-flatten.pbtxt | 2 +- .../api/golden/tensorflow.layers.-layer.pbtxt | 2 +- .../tensorflow.layers.-max-pooling1-d.pbtxt | 2 +- .../tensorflow.layers.-max-pooling2-d.pbtxt | 2 +- .../tensorflow.layers.-max-pooling3-d.pbtxt | 2 +- ...tensorflow.layers.-separable-conv2-d.pbtxt | 2 +- ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt | 2 +- ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt | 2 +- ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt | 2 +- ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt | 2 +- .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt | 2 +- ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt | 7 +- ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt | 2 +- .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt | 2 +- ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt | 2 +- 116 files changed, 237 insertions(+), 269 deletions(-) diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py index 8349188f6f..6b6cdfa242 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py @@ -40,7 +40,6 @@ from tensorflow.python.ops import rnn_cell_impl from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test -from tensorflow.python.framework import test_util # pylint: enable=protected-access diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py index 12def6dcc8..9cea2ec79a 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py @@ -169,7 +169,7 @@ class RNNTest(test.TestCase): self.assertEqual(out.get_shape(), inp.get_shape()) self.assertEqual(out.dtype, inp.dtype) - with self.test_session(use_gpu=False) as sess: + with self.test_session(use_gpu=True) as sess: input_value = np.random.randn(batch_size, input_size) values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) @@ -204,7 +204,7 @@ class RNNTest(test.TestCase): self.assertEqual(out.get_shape().as_list(), inp.get_shape().as_list()) self.assertEqual(out.dtype, inp.dtype) - with self.test_session(use_gpu=False) as sess: + with self.test_session(use_gpu=True) as sess: input_value = np.random.randn(batch_size, input_size) values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) full_dropout_values = sess.run(dropped_outputs, @@ -215,7 +215,7 @@ class RNNTest(test.TestCase): for d_v in full_dropout_values[:-1]: # Add 1.0 to dropped_out (all zeros) self.assertAllClose(d_v, np.ones_like(input_value)) - def _testDynamicCalculation(self, use_gpu): + def testDynamicCalculation(self): cell = Plus1RNNCell() sequence_length = array_ops.placeholder(dtypes.int64) batch_size = 2 @@ -230,7 +230,7 @@ class RNNTest(test.TestCase): cell, inputs, sequence_length=sequence_length, dtype=dtypes.float32) self.assertEqual(len(dynamic_outputs), len(inputs)) - with self.test_session(use_gpu=use_gpu) as sess: + with self.test_session(use_gpu=True) as sess: input_value = np.random.randn(batch_size, input_size) dynamic_values = sess.run( dynamic_outputs, @@ -261,10 +261,6 @@ class RNNTest(test.TestCase): np.vstack((1.0 * (1 + 1) * np.ones((input_size)), 1.0 * (2 + 1) * np.ones((input_size))))) - def testDynamicCalculation(self): - self._testDynamicCalculation(True) - self._testDynamicCalculation(False) - def _testScope(self, factory, prefix="prefix", use_outer_scope=True): with self.test_session(use_gpu=True, graph=ops_lib.Graph()): if use_outer_scope: @@ -309,12 +305,12 @@ class LSTMTest(test.TestCase): self._seed = 23489 np.random.seed(self._seed) - def _testNoProjNoSharding(self, use_gpu): + def testNoProjNoSharding(self): num_units = 3 input_size = 5 batch_size = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) cell = rnn_cell.LSTMCell( @@ -332,12 +328,12 @@ class LSTMTest(test.TestCase): input_value = np.random.randn(batch_size, input_size) sess.run(outputs, feed_dict={inputs[0]: input_value}) - def _testCellClipping(self, use_gpu): + def testCellClipping(self): num_units = 3 input_size = 5 batch_size = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) cell = rnn_cell.LSTMCell( @@ -363,12 +359,12 @@ class LSTMTest(test.TestCase): # if cell c is clipped to 0, tanh(c) = 0 => m==0 self.assertAllEqual(value, np.zeros((batch_size, num_units))) - def _testNoProjNoShardingSimpleStateSaver(self, use_gpu): + def testNoProjNoShardingSimpleStateSaver(self): num_units = 3 input_size = 5 batch_size = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) state_saver = TestStateSaver(batch_size, 2 * num_units) @@ -493,13 +489,13 @@ class LSTMTest(test.TestCase): self.assertAllEqual(last_states[i], named_saved_states[flat_state_names[i]]) - def _testProjNoSharding(self, use_gpu): + def testProjNoSharding(self): num_units = 3 input_size = 5 batch_size = 2 num_proj = 4 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) inputs = max_length * [ @@ -584,7 +580,7 @@ class LSTMTest(test.TestCase): state_tuple_v = sess.run(state_tuple, feed_dict={inputs[0]: input_value}) self.assertAllEqual(state_notuple_v, np.hstack(state_tuple_v)) - def _testProjSharding(self, use_gpu): + def testProjSharding(self): num_units = 3 input_size = 5 batch_size = 2 @@ -592,7 +588,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) @@ -618,7 +614,7 @@ class LSTMTest(test.TestCase): input_value = np.random.randn(batch_size, input_size) sess.run(outputs, feed_dict={inputs[0]: input_value}) - def _testDoubleInput(self, use_gpu): + def testDoubleInput(self): num_units = 3 input_size = 5 batch_size = 2 @@ -626,7 +622,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer(-1, 1, seed=self._seed) inputs = max_length * [ array_ops.placeholder( @@ -655,7 +651,7 @@ class LSTMTest(test.TestCase): values = sess.run(outputs, feed_dict={inputs[0]: input_value}) self.assertEqual(values[0].dtype, input_value.dtype) - def _testShardNoShardEquivalentOutput(self, use_gpu): + def testShardNoShardEquivalentOutput(self): num_units = 3 input_size = 5 batch_size = 2 @@ -663,7 +659,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: inputs = max_length * [ array_ops.placeholder( dtypes.float32, shape=(None, input_size)) @@ -710,7 +706,7 @@ class LSTMTest(test.TestCase): for (s_noshard, s_shard) in zip(state_values_noshard, state_values_shard): self.assertAllClose(s_noshard, s_shard, atol=1e-3) - def _testDoubleInputWithDropoutAndDynamicCalculation(self, use_gpu): + def testDoubleInputWithDropoutAndDynamicCalculation(self): """Smoke test for using LSTM with doubles, dropout, dynamic calculation.""" num_units = 3 @@ -720,7 +716,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: sequence_length = array_ops.placeholder(dtypes.int64) initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) @@ -845,38 +841,6 @@ class LSTMTest(test.TestCase): for out0, out1 in zip(outputs0_values, outputs1_values): self.assertAllEqual(out0, out1) - def testNoProjNoShardingSimpleStateSaver(self): - self._testNoProjNoShardingSimpleStateSaver(use_gpu=False) - self._testNoProjNoShardingSimpleStateSaver(use_gpu=True) - - def testNoProjNoSharding(self): - self._testNoProjNoSharding(use_gpu=False) - self._testNoProjNoSharding(use_gpu=True) - - def testCellClipping(self): - self._testCellClipping(use_gpu=False) - self._testCellClipping(use_gpu=True) - - def testProjNoSharding(self): - self._testProjNoSharding(use_gpu=False) - self._testProjNoSharding(use_gpu=True) - - def testProjSharding(self): - self._testProjSharding(use_gpu=False) - self._testProjSharding(use_gpu=True) - - def testShardNoShardEquivalentOutput(self): - self._testShardNoShardEquivalentOutput(use_gpu=False) - self._testShardNoShardEquivalentOutput(use_gpu=True) - - def testDoubleInput(self): - self._testDoubleInput(use_gpu=False) - self._testDoubleInput(use_gpu=True) - - def testDoubleInputWithDropoutAndDynamicCalculation(self): - self._testDoubleInputWithDropoutAndDynamicCalculation(use_gpu=False) - self._testDoubleInputWithDropoutAndDynamicCalculation(use_gpu=True) - def testDynamicRNNAllowsUnknownTimeDimension(self): inputs = array_ops.placeholder(dtypes.float32, shape=[1, None, 20]) cell = rnn_cell.GRUCell(30) @@ -1052,7 +1016,7 @@ class LSTMTest(test.TestCase): state_dynamic = [s.numpy() for s in nest.flatten(state_dynamic)] self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic)) - def _testDynamicEquivalentToStaticRNN(self, use_gpu, use_sequence_length): + def _testDynamicEquivalentToStaticRNN(self, use_sequence_length): time_steps = 8 num_units = 3 num_proj = 4 @@ -1083,7 +1047,7 @@ class LSTMTest(test.TestCase): state_is_tuple=False) ########### Step 1: Run static graph and generate readouts - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: if in_graph_mode: concat_inputs = array_ops.placeholder( dtypes.float32, shape=(time_steps, batch_size, input_size)) @@ -1143,7 +1107,7 @@ class LSTMTest(test.TestCase): static_individual_variable_gradients, feed_dict=feeds) ########## Step 2: Run dynamic graph and generate readouts - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: if in_graph_mode: concat_inputs = array_ops.placeholder( dtypes.float32, shape=(time_steps, batch_size, input_size)) @@ -1249,14 +1213,8 @@ class LSTMTest(test.TestCase): @test_util.run_in_graph_and_eager_modes() def testDynamicEquivalentToStaticRNN(self): - self._testDynamicEquivalentToStaticRNN( - use_gpu=False, use_sequence_length=False) - self._testDynamicEquivalentToStaticRNN( - use_gpu=True, use_sequence_length=False) - self._testDynamicEquivalentToStaticRNN( - use_gpu=False, use_sequence_length=True) - self._testDynamicEquivalentToStaticRNN( - use_gpu=True, use_sequence_length=True) + self._testDynamicEquivalentToStaticRNN(use_sequence_length=False) + self._testDynamicEquivalentToStaticRNN(use_sequence_length=False) class BidirectionalRNNTest(test.TestCase): @@ -1266,7 +1224,6 @@ class BidirectionalRNNTest(test.TestCase): np.random.seed(self._seed) def _createBidirectionalRNN(self, - use_gpu, use_shape, use_sequence_length, scope=None): @@ -1305,10 +1262,10 @@ class BidirectionalRNNTest(test.TestCase): return input_value, inputs, outputs, state_fw, state_bw, sequence_length - def _testBidirectionalRNN(self, use_gpu, use_shape): - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + def _testBidirectionalRNN(self, use_shape): + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createBidirectionalRNN(use_gpu, use_shape, True)) + self._createBidirectionalRNN(use_shape, True)) variables_lib.global_variables_initializer().run() # Run with pre-specified sequence length of 2, 3 out, s_fw, s_bw = sess.run( @@ -1350,10 +1307,10 @@ class BidirectionalRNNTest(test.TestCase): # exactly the same self.assertAllClose(s_fw, s_bw) - def _testBidirectionalRNNWithoutSequenceLength(self, use_gpu, use_shape): - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + def _testBidirectionalRNNWithoutSequenceLength(self, use_shape): + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: input_value, inputs, outputs, state_fw, state_bw, _ = ( - self._createBidirectionalRNN(use_gpu, use_shape, False)) + self._createBidirectionalRNN(use_shape, False)) variables_lib.global_variables_initializer().run() out, s_fw, s_bw = sess.run([outputs, state_fw, state_bw], feed_dict={inputs[0]: input_value}) @@ -1380,23 +1337,14 @@ class BidirectionalRNNTest(test.TestCase): self.assertAllClose(s_fw, s_bw) def testBidirectionalRNN(self): - self._testBidirectionalRNN(use_gpu=False, use_shape=False) - self._testBidirectionalRNN(use_gpu=True, use_shape=False) - self._testBidirectionalRNN(use_gpu=False, use_shape=True) - self._testBidirectionalRNN(use_gpu=True, use_shape=True) + self._testBidirectionalRNN(use_shape=False) + self._testBidirectionalRNN(use_shape=True) def testBidirectionalRNNWithoutSequenceLength(self): - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=False, use_shape=False) - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=True, use_shape=False) - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=False, use_shape=True) - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=True, use_shape=True) + self._testBidirectionalRNNWithoutSequenceLength(use_shape=False) + self._testBidirectionalRNNWithoutSequenceLength(use_shape=True) def _createBidirectionalDynamicRNN(self, - use_gpu, use_shape, use_state_tuple, use_time_major, @@ -1444,11 +1392,11 @@ class BidirectionalRNNTest(test.TestCase): return input_value, inputs, outputs, state_fw, state_bw, sequence_length - def _testBidirectionalDynamicRNN(self, use_gpu, use_shape, use_state_tuple, + def _testBidirectionalDynamicRNN(self, use_shape, use_state_tuple, use_time_major, use_sequence_length): - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createBidirectionalDynamicRNN(use_gpu, use_shape, + self._createBidirectionalDynamicRNN(use_shape, use_state_tuple, use_time_major, use_sequence_length)) variables_lib.global_variables_initializer().run() @@ -1513,14 +1461,13 @@ class BidirectionalRNNTest(test.TestCase): def testBidirectionalDynamicRNN(self): # Generate 2^5 option values # from [True, True, True, True, True] to [False, False, False, False, False] - options = itertools.product([True, False], repeat=5) + options = itertools.product([True, False], repeat=4) for option in options: self._testBidirectionalDynamicRNN( - use_gpu=option[0], - use_shape=option[1], - use_state_tuple=option[2], - use_time_major=option[3], - use_sequence_length=option[4]) + use_shape=option[0], + use_state_tuple=option[1], + use_time_major=option[2], + use_sequence_length=option[3]) def _testScope(self, factory, prefix="prefix", use_outer_scope=True): # REMARKS: factory(scope) is a function accepting a scope @@ -1549,7 +1496,7 @@ class BidirectionalRNNTest(test.TestCase): def factory(scope): return self._createBidirectionalRNN( - use_gpu=True, use_shape=True, use_sequence_length=True, scope=scope) + use_shape=True, use_sequence_length=True, scope=scope) self._testScope(factory, use_outer_scope=True) self._testScope(factory, use_outer_scope=False) @@ -1561,7 +1508,6 @@ class BidirectionalRNNTest(test.TestCase): def factory(scope): return self._createBidirectionalDynamicRNN( - use_gpu=True, use_shape=True, use_state_tuple=True, use_sequence_length=True, @@ -1839,7 +1785,7 @@ class GRUTest(test.TestCase): self._seed = 23489 np.random.seed(self._seed) - def _testDynamic(self, use_gpu): + def testDynamic(self): time_steps = 8 num_units = 3 input_size = 5 @@ -1849,7 +1795,7 @@ class GRUTest(test.TestCase): sequence_length = np.random.randint(0, time_steps, size=batch_size) - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: concat_inputs = array_ops.placeholder( dtypes.float32, shape=(time_steps, batch_size, input_size)) @@ -1870,10 +1816,6 @@ class GRUTest(test.TestCase): sess.run([outputs_dynamic, state_dynamic], feed_dict=feeds) - def testDynamic(self): - self._testDynamic(use_gpu=False) - self._testDynamic(use_gpu=True) - def _testScope(self, factory, prefix="prefix", use_outer_scope=True): with self.test_session(use_gpu=True, graph=ops_lib.Graph()): if use_outer_scope: diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index 91e18b2ba5..134d4fc8e2 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -413,7 +413,8 @@ class Layer(object): def add_variable(self, name, shape, dtype=None, initializer=None, regularizer=None, - trainable=True, constraint=None): + trainable=True, constraint=None, + partitioner=None): """Adds a new variable to the layer, or gets an existing one; returns it. Arguments: @@ -426,9 +427,19 @@ class Layer(object): "trainable_variables" (e.g. variables, biases) or "non_trainable_variables" (e.g. BatchNorm mean, stddev). constraint: constraint instance (callable). + partitioner: (optional) partitioner instance (callable). If + provided, when the requested variable is created it will be split + into multiple partitions according to `partitioner`. In this case, + an instance of `PartitionedVariable` is returned. Available + partitioners include `tf.fixed_size_partitioner` and + `tf.variable_axis_size_partitioner`. For more details, see the + documentation of `tf.get_variable` and the "Variable Partitioners + and Sharding" section of the API guide. Returns: - The created variable. + The created variable. Usually either a `Variable` or `ResourceVariable` + instance. If `partitioner` is not `None`, a `PartitionedVariable` + instance is returned. Raises: RuntimeError: If called in Eager mode with regularizers. @@ -455,7 +466,8 @@ class Layer(object): initializer=initializer, dtype=dtypes.as_dtype(dtype), constraint=constraint, - trainable=trainable and self.trainable) + trainable=trainable and self.trainable, + partitioner=partitioner) if variable in existing_variables: return variable if regularizer: diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index 1825e98259..b90c757095 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -455,6 +455,10 @@ class BasicLSTMCell(_LayerRNNCell): if not state_is_tuple: logging.warn("%s: Using a concatenated state is slower and will soon be " "deprecated. Use state_is_tuple=True.", self) + + # Inputs must be 2-dimensional. + self.input_spec = base_layer.InputSpec(ndim=2) + self._num_units = num_units self._forget_bias = forget_bias self._state_is_tuple = state_is_tuple @@ -471,9 +475,6 @@ class BasicLSTMCell(_LayerRNNCell): return self._num_units def build(self, inputs_shape): - if inputs_shape.ndims != 2: - raise ValueError("Expected inputs.shape to be rank 2, saw shape: %s" - % inputs_shape) if inputs_shape[1].value is None: raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" % inputs_shape) @@ -537,7 +538,7 @@ class BasicLSTMCell(_LayerRNNCell): return new_h, new_state -class LSTMCell(RNNCell): +class LSTMCell(_LayerRNNCell): """Long short-term memory unit (LSTM) recurrent network cell. The default non-peephole implementation is based on: @@ -564,7 +565,7 @@ class LSTMCell(RNNCell): initializer=None, num_proj=None, proj_clip=None, num_unit_shards=None, num_proj_shards=None, forget_bias=1.0, state_is_tuple=True, - activation=None, reuse=None): + activation=None, reuse=None, name=None): """Initialize the parameters for an LSTM cell. Args: @@ -594,11 +595,14 @@ class LSTMCell(RNNCell): reuse: (optional) Python boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. + name: String, the name of the layer. Layers with the same name will + share weights, but to avoid mistakes we require reuse=True in such + cases. - When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. + When restoring from CudnnLSTM-trained checkpoints, use + `CudnnCompatibleLSTMCell` instead. """ - super(LSTMCell, self).__init__(_reuse=reuse) + super(LSTMCell, self).__init__(_reuse=reuse, name=name) if not state_is_tuple: logging.warn("%s: Using a concatenated state is slower and will soon be " "deprecated. Use state_is_tuple=True.", self) @@ -608,6 +612,9 @@ class LSTMCell(RNNCell): "deprecated and will be removed in Jan 2017. " "Use a variable scope with a partitioner instead.", self) + # Inputs must be 2-dimensional. + self.input_spec = base_layer.InputSpec(ndim=2) + self._num_units = num_units self._use_peepholes = use_peepholes self._cell_clip = cell_clip @@ -630,12 +637,6 @@ class LSTMCell(RNNCell): LSTMStateTuple(num_units, num_units) if state_is_tuple else 2 * num_units) self._output_size = num_units - self._linear1 = None - self._linear2 = None - if self._use_peepholes: - self._w_f_diag = None - self._w_i_diag = None - self._w_o_diag = None @property def state_size(self): @@ -645,6 +646,47 @@ class LSTMCell(RNNCell): def output_size(self): return self._output_size + def build(self, inputs_shape): + if inputs_shape[1].value is None: + raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" + % inputs_shape) + + input_depth = inputs_shape[1].value + h_depth = self._num_units if self._num_proj is None else self._num_proj + maybe_partitioner = ( + partitioned_variables.fixed_size_partitioner(self._num_unit_shards) + if self._num_unit_shards is not None + else None) + self._kernel = self.add_variable( + _WEIGHTS_VARIABLE_NAME, + shape=[input_depth + h_depth, 4 * self._num_units], + initializer=self._initializer, + partitioner=maybe_partitioner) + self._bias = self.add_variable( + _BIAS_VARIABLE_NAME, + shape=[4 * self._num_units], + initializer=init_ops.constant_initializer(0.0, dtype=self.dtype)) + if self._use_peepholes: + self._w_f_diag = self.add_variable("w_f_diag", shape=[self._num_units], + initializer=self._initializer) + self._w_i_diag = self.add_variable("w_i_diag", shape=[self._num_units], + initializer=self._initializer) + self._w_o_diag = self.add_variable("w_o_diag", shape=[self._num_units], + initializer=self._initializer) + + if self._num_proj is not None: + maybe_proj_partitioner = ( + partitioned_variables.fixed_size_partitioner(self._num_proj_shards) + if self._num_proj_shards is not None + else None) + self._proj_kernel = self.add_variable( + "projection/%s" % _WEIGHTS_VARIABLE_NAME, + shape=[self._num_units, self._num_proj], + initializer=self._initializer, + partitioner=maybe_proj_partitioner) + + self._built = True + def call(self, inputs, state): """Run one step of LSTM. @@ -679,37 +721,18 @@ class LSTMCell(RNNCell): c_prev = array_ops.slice(state, [0, 0], [-1, self._num_units]) m_prev = array_ops.slice(state, [0, self._num_units], [-1, num_proj]) - dtype = inputs.dtype input_size = inputs.get_shape().with_rank(2)[1] if input_size.value is None: raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - if self._linear1 is None: - scope = vs.get_variable_scope() - with vs.variable_scope( - scope, initializer=self._initializer) as unit_scope: - if self._num_unit_shards is not None: - unit_scope.set_partitioner( - partitioned_variables.fixed_size_partitioner( - self._num_unit_shards)) - self._linear1 = _Linear([inputs, m_prev], 4 * self._num_units, True) # i = input_gate, j = new_input, f = forget_gate, o = output_gate - lstm_matrix = self._linear1([inputs, m_prev]) + lstm_matrix = math_ops.matmul( + array_ops.concat([inputs, m_prev], 1), self._kernel) + lstm_matrix = nn_ops.bias_add(lstm_matrix, self._bias) + i, j, f, o = array_ops.split( value=lstm_matrix, num_or_size_splits=4, axis=1) # Diagonal connections - if self._use_peepholes and self._w_f_diag is None: - scope = vs.get_variable_scope() - with vs.variable_scope( - scope, initializer=self._initializer) as unit_scope: - with vs.variable_scope(unit_scope): - self._w_f_diag = vs.get_variable( - "w_f_diag", shape=[self._num_units], dtype=dtype) - self._w_i_diag = vs.get_variable( - "w_i_diag", shape=[self._num_units], dtype=dtype) - self._w_o_diag = vs.get_variable( - "w_o_diag", shape=[self._num_units], dtype=dtype) - if self._use_peepholes: c = (sigmoid(f + self._forget_bias + self._w_f_diag * c_prev) * c_prev + sigmoid(i + self._w_i_diag * c_prev) * self._activation(j)) @@ -727,16 +750,7 @@ class LSTMCell(RNNCell): m = sigmoid(o) * self._activation(c) if self._num_proj is not None: - if self._linear2 is None: - scope = vs.get_variable_scope() - with vs.variable_scope(scope, initializer=self._initializer): - with vs.variable_scope("projection") as proj_scope: - if self._num_proj_shards is not None: - proj_scope.set_partitioner( - partitioned_variables.fixed_size_partitioner( - self._num_proj_shards)) - self._linear2 = _Linear(m, self._num_proj, False) - m = self._linear2(m) + m = math_ops.matmul(m, self._proj_kernel) if self._proj_clip is not None: # pylint: disable=invalid-unary-operand-type diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt index c3d8893317..38e6128644 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt index ea59596431..0fa6064661 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt index 7e9b6bd70a..75d56bf445 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt index 804fb45784..6e52b6238d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt index 6577856383..0e16774e86 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt index fc4452948a..98112762cf 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt index ce19cea7ca..2e093c0359 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt index 2ea54c2e31..bada65e2f9 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt index 6fa1e153e0..120807c4b5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt index c6ff50bffc..834365f0f7 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt index 6d90a59d1e..462a52ec1e 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt index 278e5b583d..b802b363d0 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt index c9991db5c9..5279b2ab17 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt @@ -107,7 +107,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt index ec3c43945f..b800eb9796 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt index 2d6560828e..a0906e62cf 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt index f6f77ff805..47c63c1157 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt index 854a06bf56..e90b90e801 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt index 5e71a9d355..aa571b722d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt index e7c98913fb..911c73f846 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt index 3c4d078d1e..bb111b327c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt index 8043eb0610..5a5ec635cc 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt index a9a90891a4..190b670fa2 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt index dae5a66190..a26ec82f2b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt index 37aa80eb70..19b5bdf36b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt index fa28ce17ec..773ef01feb 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt index 8e2b530d08..3a67ac00ab 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt index 70b1c50a0a..de5a695b69 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt index 1b2b4e934d..bf251b4df5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt index fb0fcd2614..92a74cec68 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt index af8ad3abaa..cdd62eee0d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt index e774a4d412..7935143b2c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt index 46eb767208..497eb00499 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt index 5e74cb6970..35616cbebb 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt index a4c8759a2c..427c6fde90 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt index 9738dd004a..9237399254 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt index ce033eaa00..1428691afe 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt index 4cd6d714a0..655734cc43 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt index 2bd80f97ae..d97f06ea13 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt index a9d00fd7c1..52886b2106 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt index a2b00778fe..ccb6459357 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt index 01a9839ccc..1f25eb1cc6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt index b041dfc71e..a37d6dda28 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt index 6ba06a4e7e..9f276fd547 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt index fb62a3e035..eaa9b477d8 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt index 3d1c66441c..f4d37a5f63 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt index d55a82e0a3..afddd2d4cb 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt index 70177c8623..12cd49c955 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt index da231a4fce..146241c172 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt index aa3eb1c704..00475301aa 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt index 40f0f7c800..b2df5fba8f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt index 1a9ec4a506..20935e2f99 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt index 69086963b6..59508c2f11 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt index d350a52171..ca904a2b8c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt @@ -101,7 +101,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt index 05952c1d96..f52fd02515 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt index c49b8de5fb..b5c32d1cdf 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt index e24e3697b2..0ac2b83a99 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt index 246340a1ce..de2a28d985 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt index eb631b1d38..130d932fd6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt index cfe6af339e..82a6f6d539 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt index 4bb5a23927..ca2fd4e502 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt index 6c9b9a92eb..885e30f879 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt index cdc4c43ad6..102879d2f5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt index 4959dc58d1..4240616146 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt index 7ff5ee02e1..4b32c2e99f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt index 860ebd509b..0c964235ae 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt index e32800bd25..797a073b8a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt index 8b453f7a1b..7dc1fa6964 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt index 9b53609e4d..dedb48151a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt index f7a774a38f..bb30c0a945 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt index 4f1d2db4cc..7867e3c1fd 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt index 066519cba8..0fb6e84f8d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt index 6a08eb785b..f4148fcc23 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt index b85003d52e..9773c4acc7 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt index 83d4258a66..d4de587a48 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt index a49060b860..af210fab8d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt index 01b91b9bbc..8cfb33a148 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt index 4713bd16e1..34c9efb3ca 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt @@ -107,7 +107,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt index 393980ecde..bb42cdcb65 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt index 7ddb282f06..6d3c2ebfef 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt index c1bd2dcbaf..d790cf2e08 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt index c020dc3954..9cee68874a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt @@ -106,7 +106,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt index b7fe482145..ba6c23ae75 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt index 51f50882b2..cb587d67b0 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt index e558931ead..415720cbe1 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt index 1f3422b9a1..af9a44086f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt @@ -120,7 +120,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt index 187c3a85b3..5034fdff2a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt @@ -133,7 +133,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt index 7fdf97ed79..6e595ca343 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt index 5911fbefa9..7b6c30773b 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt index e837458615..7a7664e800 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt index 6e07b911a4..c9f5c18f25 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt index 9ee79be96d..1fa00d7b2f 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt index 67bd7d2cc1..a92a1094ac 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt @@ -95,7 +95,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt index f310b7ea86..7fa78ab20b 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt index b786667795..e92e4859ae 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt @@ -95,7 +95,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt index 02c8130b48..87e5c2949e 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt index 268cb788d1..cc4ee4c8a5 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt index 969ec33578..99ab2ef97c 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt index fb602e41be..f4074c5a4f 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt index ec65fc4555..ec51609dee 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt @@ -92,7 +92,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt index 60aec6cd14..745c532e94 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt index bc2f49cc18..f8244c01b6 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt index 83b98059f9..df5378f279 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt index 83f3ed82da..c55d2bccc9 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt @@ -95,7 +95,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt index b8e27cc6cb..49066eecaa 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt index 29bc20ef1a..5646461b24 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt index 17ee1ff5fb..81dcd90e81 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt index fe4f630a39..8ff225897a 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt index 1c8dd65d27..2adfc747d1 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt index 0f294e216a..8d17153972 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.nn.rnn_cell.LSTMCell" tf_class { is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -90,7 +91,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'num_units\', \'use_peepholes\', \'cell_clip\', \'initializer\', \'num_proj\', \'proj_clip\', \'num_unit_shards\', \'num_proj_shards\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'1.0\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'num_units\', \'use_peepholes\', \'cell_clip\', \'initializer\', \'num_proj\', \'proj_clip\', \'num_unit_shards\', \'num_proj_shards\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'1.0\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -102,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,7 +111,7 @@ tf_class { } member_method { name: "build" - argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None" } member_method { name: "call" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt index ed42631471..68c3064dd4 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt index 2c7dc7c4f2..86ff0fee2b 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt @@ -101,7 +101,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt index dbcbf29586..1a6f8a3b7d 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" -- GitLab From 966016b7f2382658e7c84baae0596d35f0a49bae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 17:16:35 -0700 Subject: [PATCH 438/573] BUILD dependency cleanup in contrib/... PiperOrigin-RevId: 173613863 --- tensorflow/contrib/BUILD | 2 ++ tensorflow/contrib/all_reduce/BUILD | 3 ++- tensorflow/contrib/bayesflow/BUILD | 4 ++-- .../boosted_trees/estimator_batch/BUILD | 2 ++ .../contrib/data/python/kernel_tests/BUILD | 3 --- tensorflow/contrib/data/python/ops/BUILD | 6 ++--- tensorflow/contrib/distributions/BUILD | 8 +++++++ tensorflow/contrib/eager/python/BUILD | 24 +++++++++++++++---- tensorflow/contrib/estimator/BUILD | 3 ++- tensorflow/contrib/framework/BUILD | 3 +-- tensorflow/contrib/gan/BUILD | 7 +++++- tensorflow/contrib/gdr/BUILD | 1 - tensorflow/contrib/graph_editor/BUILD | 2 +- tensorflow/contrib/hooks/BUILD | 1 + tensorflow/contrib/image/BUILD | 3 ++- .../contrib/kfac/python/kernel_tests/BUILD | 1 - tensorflow/contrib/kfac/python/ops/BUILD | 2 ++ tensorflow/contrib/labeled_tensor/BUILD | 2 +- tensorflow/contrib/layers/BUILD | 9 +++---- 19 files changed, 58 insertions(+), 28 deletions(-) diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index ee3dd5079e..2e9b96bb1d 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -88,8 +88,10 @@ py_library( "//tensorflow/contrib/tfprof", "//tensorflow/contrib/timeseries", "//tensorflow/contrib/tpu", + "//tensorflow/contrib/tpu:tpu_py", "//tensorflow/contrib/training:training_py", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:util", ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_ops_py"]), ) diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD index 744ae4c1f4..35b9de27e7 100644 --- a/tensorflow/contrib/all_reduce/BUILD +++ b/tensorflow/contrib/all_reduce/BUILD @@ -19,9 +19,10 @@ py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - "//tensorflow/contrib/nccl:nccl_ops", + "//tensorflow/contrib/nccl:nccl_py", "//tensorflow/python:array_ops", "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", ], ) diff --git a/tensorflow/contrib/bayesflow/BUILD b/tensorflow/contrib/bayesflow/BUILD index 324e519a6d..8bb742d289 100644 --- a/tensorflow/contrib/bayesflow/BUILD +++ b/tensorflow/contrib/bayesflow/BUILD @@ -20,8 +20,9 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:functional_ops", + "//tensorflow/python:gradients", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", @@ -31,7 +32,6 @@ py_library( "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", "//tensorflow/python/ops/distributions", "//third_party/py/numpy", "@six_archive//:six", diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD index d0ee1fd60d..7792c7127c 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD +++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD @@ -124,6 +124,8 @@ py_library( srcs_version = "PY2AND3", deps = [ ":model", + "//tensorflow/contrib/boosted_trees:losses", "//tensorflow/contrib/learn", + "//tensorflow/python:math_ops", ], ) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 36af55a7ec..c310e79741 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -302,11 +302,8 @@ py_test( "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", "//tensorflow/python:string_ops", - "//tensorflow/python:training", "//tensorflow/python:util", - "//tensorflow/python:variables", "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index b17b02ee35..a6eb50014a 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -12,9 +12,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":transformation_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:script_ops", - "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/util:nest", ], @@ -48,6 +46,7 @@ py_library( "//tensorflow/python:platform", "//tensorflow/python:sparse_tensor", "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/ops:readers", "//tensorflow/python/data/util:nest", @@ -76,7 +75,6 @@ py_library( "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", - "//tensorflow/python:resource_variable_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", "//tensorflow/python/data/ops:dataset_ops", diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index 1305c28012..bc72bc37a7 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -18,14 +18,20 @@ py_library( "//tensorflow/contrib/linalg:linalg_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", + "//tensorflow/python:clip_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn_ops", + "//tensorflow/python:template", "//tensorflow/python:tensor_util", "//tensorflow/python:util", + "//tensorflow/python:variable_scope", "//tensorflow/python/ops/distributions", + "//tensorflow/python/ops/linalg", "//third_party/py/numpy", ], ) @@ -55,7 +61,9 @@ py_library( "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", "//tensorflow/python/ops/distributions", + "//tensorflow/python/ops/linalg", "//third_party/py/numpy", "@six_archive//:six", ], diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 179c27ba80..cb7b5cf462 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -18,6 +18,7 @@ py_library( ":saver", ":summary_writer", "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:numerics", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", @@ -52,6 +53,7 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python/data/util:nest", "//tensorflow/python/eager:context", @@ -64,10 +66,11 @@ py_test( srcs_version = "PY2AND3", deps = [ ":datasets", + "//tensorflow/python:dtypes", "//tensorflow/python:math_ops", + "//tensorflow/python:script_ops", "//tensorflow/python/data", "//tensorflow/python/eager:test", - "//third_party/py/numpy", ], ) @@ -76,7 +79,11 @@ py_library( srcs = ["saver.py"], srcs_version = "PY2AND3", deps = [ + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:training", + "//tensorflow/python/eager:context", ], ) @@ -100,12 +107,14 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/summary:gen_summary_ops", - "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:init_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:state_ops", "//tensorflow/python:summary_op_util", - "//tensorflow/python:training", + "//tensorflow/python:variable_scope", "//tensorflow/python/eager:context", ], ) @@ -137,8 +146,7 @@ py_library( "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:layers_base", + "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", @@ -154,8 +162,13 @@ py_test( srcs_version = "PY2AND3", deps = [ ":metrics", + "//tensorflow/contrib/summary:summary_ops", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", + "//tensorflow/python:lib", + "//tensorflow/python:platform", + "//tensorflow/python:training", "//tensorflow/python:variables", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", @@ -203,6 +216,7 @@ py_library( "//tensorflow/python:layers_base", "//tensorflow/python:variable_scope", "//tensorflow/python/estimator:util", + "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 8a7d67b5c2..79b166ac88 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -190,7 +190,8 @@ py_test( deps = [ ":logit_fns", "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:session", "//tensorflow/python/estimator:model_fn", ], ) diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD index dd882acb8e..90aed3065b 100644 --- a/tensorflow/contrib/framework/BUILD +++ b/tensorflow/contrib/framework/BUILD @@ -239,7 +239,6 @@ py_test( deps = [ ":framework_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", @@ -247,6 +246,7 @@ py_test( "//tensorflow/python:nn_ops", "//tensorflow/python:partitioned_variables", "//tensorflow/python:platform", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", @@ -279,7 +279,6 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", - "//tensorflow/python:errors", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:partitioned_variables", diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD index 27a5d6ec31..1418c87023 100644 --- a/tensorflow/contrib/gan/BUILD +++ b/tensorflow/contrib/gan/BUILD @@ -202,6 +202,7 @@ py_library( "//tensorflow/python:embedding_ops", "//tensorflow/python:math_ops", "//tensorflow/python:tensor_util", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", ], ) @@ -234,6 +235,7 @@ py_library( "//tensorflow/python:nn", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", ], ) @@ -267,7 +269,10 @@ py_library( "python/features/python/clip_weights_impl.py", ], srcs_version = "PY2AND3", - deps = ["//tensorflow/contrib/opt:opt_py"], + deps = [ + "//tensorflow/contrib/opt:opt_py", + "//tensorflow/python:util", + ], ) py_test( diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD index bebcf079ba..a8053be69b 100644 --- a/tensorflow/contrib/gdr/BUILD +++ b/tensorflow/contrib/gdr/BUILD @@ -119,7 +119,6 @@ cc_library( ":gdr_memory_manager", ":gdr_rendezvous_mgr", ":gdr_worker", - "//tensorflow/core:lib_internal", "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib", ], alwayslink = 1, diff --git a/tensorflow/contrib/graph_editor/BUILD b/tensorflow/contrib/graph_editor/BUILD index b4c53d3da6..967ad2fc09 100644 --- a/tensorflow/contrib/graph_editor/BUILD +++ b/tensorflow/contrib/graph_editor/BUILD @@ -144,12 +144,12 @@ py_test( ":graph_editor_py", ":match", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:gradients", "//tensorflow/python:math_ops", + "//tensorflow/python:session", "//tensorflow/python:variables", "//third_party/py/numpy", ], diff --git a/tensorflow/contrib/hooks/BUILD b/tensorflow/contrib/hooks/BUILD index 1576c9ec9b..1b528d7afc 100644 --- a/tensorflow/contrib/hooks/BUILD +++ b/tensorflow/contrib/hooks/BUILD @@ -20,6 +20,7 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/python:training", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD index d0600d4668..c0c56d2e4a 100755 --- a/tensorflow/contrib/image/BUILD +++ b/tensorflow/contrib/image/BUILD @@ -143,12 +143,13 @@ py_library( srcs_version = "PY2AND3", deps = [ ":distort_image_ops", + ":single_image_random_dot_stereograms_py", "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:image_ops", "//tensorflow/python:platform", "//tensorflow/python:random_ops", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD index 5d86373a23..0653e71d12 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD +++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD @@ -88,7 +88,6 @@ py_test( deps = [ "//tensorflow/contrib/kfac/python/ops:kfac_optimizer", "//tensorflow/contrib/kfac/python/ops:layer_collection", - "//tensorflow/contrib/kfac/python/ops:loss_functions", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_ops", diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD index 5d5046c9ec..de4b8920b8 100644 --- a/tensorflow/contrib/kfac/python/ops/BUILD +++ b/tensorflow/contrib/kfac/python/ops/BUILD @@ -66,6 +66,7 @@ py_library( deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", "//tensorflow/python/ops/distributions", "@six_archive//:six", ], @@ -89,6 +90,7 @@ py_library( ":utils", "//tensorflow/python:gradients", "//tensorflow/python:math_ops", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD index 4eba29caec..894e6f6946 100644 --- a/tensorflow/contrib/labeled_tensor/BUILD +++ b/tensorflow/contrib/labeled_tensor/BUILD @@ -109,9 +109,9 @@ py_test( ":test_util", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", ], ) diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD index bbb4fb1f57..1ae4d281c4 100644 --- a/tensorflow/contrib/layers/BUILD +++ b/tensorflow/contrib/layers/BUILD @@ -153,10 +153,10 @@ py_test( deps = [ ":layers_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", + "//tensorflow/python:session", "//third_party/py/numpy", ], ) @@ -168,9 +168,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":layers_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", "//third_party/py/numpy", @@ -238,6 +238,7 @@ py_test( ":layers_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:lookup_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", @@ -280,9 +281,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":layers_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:variables", ], ) @@ -294,9 +295,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":layers_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//third_party/py/numpy", -- GitLab From f9aa795318e6d82c310440fb3f80b240bb034fcc Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Thu, 26 Oct 2017 17:40:41 -0700 Subject: [PATCH 439/573] Introduce TensorBoard SQL schema for summaries Unlike dataset_ops which are designed to read arbitrary SQL data into tensors, this is designed to write TensorFlow data to SQL. Please note that this code is going to be moved into the TensorBoard codebase as soon as that's feasible. PiperOrigin-RevId: 173616340 --- tensorflow/BUILD | 1 + tensorflow/contrib/tensorboard/db/BUILD | 36 ++ tensorflow/contrib/tensorboard/db/schema.cc | 412 ++++++++++++++++++ tensorflow/contrib/tensorboard/db/schema.h | 33 ++ .../contrib/tensorboard/db/schema_test.cc | 34 ++ 5 files changed, 516 insertions(+) create mode 100644 tensorflow/contrib/tensorboard/db/BUILD create mode 100644 tensorflow/contrib/tensorboard/db/schema.cc create mode 100644 tensorflow/contrib/tensorboard/db/schema.h create mode 100644 tensorflow/contrib/tensorboard/db/schema_test.cc diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 20f02ad50a..8667fd7c91 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -437,6 +437,7 @@ filegroup( "//tensorflow/contrib/tensor_forest/kernels/v4:all_files", "//tensorflow/contrib/tensor_forest/proto:all_files", "//tensorflow/contrib/tensorboard:all_files", + "//tensorflow/contrib/tensorboard/db:all_files", "//tensorflow/contrib/testing:all_files", "//tensorflow/contrib/text:all_files", "//tensorflow/contrib/tfprof:all_files", diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/contrib/tensorboard/db/BUILD new file mode 100644 index 0000000000..f056632295 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/BUILD @@ -0,0 +1,36 @@ +# Description: +# TensorBoard database code. + +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "tf_cc_test") + +cc_library( + name = "schema", + srcs = ["schema.cc"], + hdrs = ["schema.h"], + deps = [ + "//tensorflow/core:lib", + "//tensorflow/core/lib/db:sqlite", + ], +) + +tf_cc_test( + name = "schema_test", + srcs = ["schema_test.cc"], + deps = [ + ":schema", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/db:sqlite", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["*"]), + visibility = ["//tensorflow:__pkg__"], +) diff --git a/tensorflow/contrib/tensorboard/db/schema.cc b/tensorflow/contrib/tensorboard/db/schema.cc new file mode 100644 index 0000000000..f5a8e02a9b --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/schema.cc @@ -0,0 +1,412 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/schema.h" + +namespace tensorflow { +namespace db { +namespace { + +class SqliteSchema { + public: + explicit SqliteSchema(Sqlite* db) : db_(db) {} + ~SqliteSchema() { db_ = nullptr; } + + /// \brief Creates Tensors table. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// tag_id: ID of associated Tag. + /// computed_time: Float UNIX timestamp with microsecond precision. + /// In the old summaries system that uses FileWriter, this is the + /// wall time around when tf.Session.run finished. In the new + /// summaries system, it is the wall time of when the tensor was + /// computed. On systems with monotonic clocks, it is calculated + /// by adding the monotonic run duration to Run.started_time. + /// This field is not indexed because, in practice, it should be + /// ordered the same or nearly the same as TensorIndex, so local + /// insertion sort might be more suitable. + /// step: User-supplied number, ordering this tensor in Tag. + /// If NULL then the Tag must have only one Tensor. + /// tensor: Can be an INTEGER (DT_INT64), FLOAT (DT_DOUBLE), or + /// BLOB. The structure of a BLOB is currently undefined, but in + /// essence it is a Snappy tf.TensorProto that spills over into + /// TensorChunks. + Status CreateTensorsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Tensors ( + rowid INTEGER PRIMARY KEY, + tag_id INTEGER NOT NULL, + computed_time REAL, + step INTEGER, + tensor BLOB + ) + )sql"); + } + + /// \brief Creates TensorChunks table. + /// + /// This table can be used to split up a tensor across many rows, + /// which has the advantage of not slowing down table scans on the + /// main table, allowing asynchronous fetching, minimizing copying, + /// and preventing large buffers from being allocated. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// tag_id: ID of associated Tag. + /// step: Same as corresponding Tensors.step. + /// sequence: 1-indexed sequence number for ordering chunks. Please + /// note that the 0th index is Tensors.tensor. + /// chunk: Bytes of next chunk in tensor. + Status CreateTensorChunksTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS TensorChunks ( + rowid INTEGER PRIMARY KEY, + tag_id INTEGER NOT NULL, + step INTEGER, + sequence INTEGER, + chunk BLOB + ) + )sql"); + } + + /// \brief Creates Tags table. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// tag_id: Permanent >0 unique ID. + /// run_id: Optional ID of associated Run. + /// tag_name: The tag field in summary.proto, unique across Run. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the wall time of when the row was inserted into the + /// DB. It may be used as a hint for an archival job. + /// metadata: Optional BLOB of SummaryMetadata proto. + /// display_name: Optional for GUI and defaults to tag_name. + /// summary_description: Optional markdown information. + Status CreateTagsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Tags ( + rowid INTEGER PRIMARY KEY, + run_id INTEGER, + tag_id INTEGER NOT NULL, + tag_name TEXT, + inserted_time DOUBLE, + metadata BLOB, + display_name TEXT, + description TEXT + ) + )sql"); + } + + /// \brief Creates Runs table. + /// + /// This table stores information about runs. Each row usually + /// represents a single attempt at training or testing a TensorFlow + /// model, with a given set of hyper-parameters, whose summaries are + /// written out to a single event logs directory with a monotonic step + /// counter. + /// + /// When a run is deleted from this table, TensorBoard should treat all + /// information associated with it as deleted, even if those rows in + /// different tables still exist. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// run_id: Permanent >0 unique ID. + /// experiment_id: Optional ID of associated Experiment. + /// run_name: User-supplied string, unique across Experiment. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the time the row was inserted into the database. It + /// does not change. + /// started_time: Float UNIX timestamp with µs precision. In the + /// old summaries system that uses FileWriter, this is + /// approximated as the first tf.Event.wall_time. In the new + /// summaries system, it is the wall time of when summary writing + /// started, from the perspective of whichever machine talks to + /// the database. This field will be mutated if the run is + /// restarted. + /// description: Optional markdown information. + /// graph: Snappy tf.GraphDef proto with node field cleared. That + /// field can be recreated using GraphNodes and NodeDefs. + Status CreateRunsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Runs ( + rowid INTEGER PRIMARY KEY, + experiment_id INTEGER, + run_id INTEGER NOT NULL, + run_name TEXT, + inserted_time REAL, + started_time REAL, + description TEXT, + graph BLOB + ) + )sql"); + } + + /// \brief Creates Experiments table. + /// + /// This table stores information about experiments, which are sets of + /// runs. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// user_id: Optional ID of associated User. + /// experiment_id: Permanent >0 unique ID. + /// experiment_name: User-supplied string, unique across User. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the time the row was inserted into the database. It + /// does not change. + /// started_time: Float UNIX timestamp with µs precision. This is + /// the MIN(experiment.started_time, run.started_time) of each + /// Run added to the database. + /// description: Optional markdown information. + Status CreateExperimentsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Experiments ( + rowid INTEGER PRIMARY KEY, + user_id INTEGER, + experiment_id INTEGER NOT NULL, + experiment_name TEXT, + inserted_time REAL, + started_time REAL, + description TEXT + ) + )sql"); + } + + /// \brief Creates Users table. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// user_id: Permanent >0 unique ID. + /// user_name: Unique user name. + /// email: Optional unique email address. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the time the row was inserted into the database. It + /// does not change. + Status CreateUsersTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Users ( + rowid INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL, + user_name TEXT, + email TEXT, + inserted_time REAL + ) + )sql"); + } + + /// \brief Creates NodeDefs table. + /// + /// This table stores NodeDef protos which define the GraphDef for a + /// Run. This functions like a hash table so rows can be shared by + /// multiple Runs in an Experiment. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// experiment_id: Optional int64 for grouping rows. + /// node_def_id: Permanent >0 unique ID. + /// fingerprint: Optional farmhash::Fingerprint64() of uncompressed + /// node_def bytes, coerced to int64. + /// node_def: BLOB containing a Snappy tf.NodeDef proto. + Status CreateNodeDefsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS NodeDefs ( + rowid INTEGER PRIMARY KEY, + experiment_id INTEGER, + node_def_id INTEGER NOT NULL, + fingerprint INTEGER, + node_def TEXT + ) + )sql"); + } + + /// \brief Creates RunNodeDefs table. + /// + /// Table mapping Runs to NodeDefs. This is used to recreate the node + /// field of the GraphDef proto. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// run_id: Mandatory ID of associated Run. + /// node_def_id: Mandatory ID of associated NodeDef. + Status CreateRunNodeDefsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS RunNodeDefs ( + rowid INTEGER PRIMARY KEY, + run_id INTEGER NOT NULL, + node_def_id INTEGER NOT NULL + ) + )sql"); + } + + /// \brief Uniquely indexes (tag_id, step) on Tensors table. + Status CreateTensorIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TensorIndex + ON Tensors (tag_id, step) + )sql"); + } + + /// \brief Uniquely indexes (tag_id, step, sequence) on TensorChunks table. + Status CreateTensorChunkIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TensorChunkIndex + ON TensorChunks (tag_id, step, sequence) + )sql"); + } + + /// \brief Uniquely indexes tag_id on Tags table. + Status CreateTagIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TagIdIndex + ON Tags (tag_id) + )sql"); + } + + /// \brief Uniquely indexes run_id on Runs table. + Status CreateRunIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS RunIdIndex + ON Runs (run_id) + )sql"); + } + + /// \brief Uniquely indexes experiment_id on Experiments table. + Status CreateExperimentIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS ExperimentIdIndex + ON Experiments (experiment_id) + )sql"); + } + + /// \brief Uniquely indexes user_id on Users table. + Status CreateUserIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS UserIdIndex + ON Users (user_id) + )sql"); + } + + /// \brief Uniquely indexes node_def_id on NodeDefs table. + Status CreateNodeDefIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS NodeDefIdIndex + ON NodeDefs (node_def_id) + )sql"); + } + + /// \brief Uniquely indexes (run_id, tag_name) on Tags table. + Status CreateTagNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TagNameIndex + ON Tags (run_id, tag_name) + WHERE tag_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes (experiment_id, run_name) on Runs table. + Status CreateRunNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS RunNameIndex + ON Runs (experiment_id, run_name) + WHERE run_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes (user_id, experiment_name) on Experiments table. + Status CreateExperimentNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS ExperimentNameIndex + ON Experiments (user_id, experiment_name) + WHERE experiment_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes user_name on Users table. + Status CreateUserNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS UserNameIndex + ON Users (user_name) + WHERE user_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes email on Users table. + Status CreateUserEmailIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS UserEmailIndex + ON Users (email) + WHERE email IS NOT NULL + )sql"); + } + + /// \brief Indexes (experiment_id, fingerprint) on NodeDefs table. + Status CreateNodeDefFingerprintIndex() { + return Run(R"sql( + CREATE INDEX IF NOT EXISTS NodeDefFingerprintIndex + ON NodeDefs (experiment_id, fingerprint) + WHERE fingerprint IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes (run_id, node_def_id) on RunNodeDefs table. + Status CreateRunNodeDefIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS RunNodeDefIndex + ON RunNodeDefs (run_id, node_def_id) + )sql"); + } + + Status Run(const char* sql) { + auto stmt = db_->Prepare(sql); + TF_RETURN_WITH_CONTEXT_IF_ERROR(stmt->StepAndReset(), sql); + return Status::OK(); + } + + private: + Sqlite* db_; +}; + +} // namespace + +Status SetupTensorboardSqliteDb(Sqlite* db) { + SqliteSchema s(db); + TF_RETURN_IF_ERROR(s.CreateTensorsTable()); + TF_RETURN_IF_ERROR(s.CreateTensorChunksTable()); + TF_RETURN_IF_ERROR(s.CreateTagsTable()); + TF_RETURN_IF_ERROR(s.CreateRunsTable()); + TF_RETURN_IF_ERROR(s.CreateExperimentsTable()); + TF_RETURN_IF_ERROR(s.CreateUsersTable()); + TF_RETURN_IF_ERROR(s.CreateNodeDefsTable()); + TF_RETURN_IF_ERROR(s.CreateRunNodeDefsTable()); + TF_RETURN_IF_ERROR(s.CreateTensorIndex()); + TF_RETURN_IF_ERROR(s.CreateTensorChunkIndex()); + TF_RETURN_IF_ERROR(s.CreateTagIdIndex()); + TF_RETURN_IF_ERROR(s.CreateRunIdIndex()); + TF_RETURN_IF_ERROR(s.CreateExperimentIdIndex()); + TF_RETURN_IF_ERROR(s.CreateUserIdIndex()); + TF_RETURN_IF_ERROR(s.CreateNodeDefIdIndex()); + TF_RETURN_IF_ERROR(s.CreateTagNameIndex()); + TF_RETURN_IF_ERROR(s.CreateRunNameIndex()); + TF_RETURN_IF_ERROR(s.CreateExperimentNameIndex()); + TF_RETURN_IF_ERROR(s.CreateUserNameIndex()); + TF_RETURN_IF_ERROR(s.CreateUserEmailIndex()); + TF_RETURN_IF_ERROR(s.CreateNodeDefFingerprintIndex()); + TF_RETURN_IF_ERROR(s.CreateRunNodeDefIndex()); + return Status::OK(); +} + +} // namespace db +} // namespace tensorflow diff --git a/tensorflow/contrib/tensorboard/db/schema.h b/tensorflow/contrib/tensorboard/db/schema.h new file mode 100644 index 0000000000..d3a6922d94 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/schema.h @@ -0,0 +1,33 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ +#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/db/sqlite.h" + +namespace tensorflow { +namespace db { + +/// \brief Creates TensorBoard SQLite tables and indexes. +/// +/// If they are already created, this has no effect. If schema +/// migrations are necessary, they will be performed with logging. +Status SetupTensorboardSqliteDb(Sqlite* db); + +} // namespace db +} // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ diff --git a/tensorflow/contrib/tensorboard/db/schema_test.cc b/tensorflow/contrib/tensorboard/db/schema_test.cc new file mode 100644 index 0000000000..a4302dda44 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/schema_test.cc @@ -0,0 +1,34 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/schema.h" + +#include + +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace db { +namespace { + +TEST(SchemaTest, SmokeTestTensorboardSchema) { + std::unique_ptr db; + TF_ASSERT_OK(Sqlite::Open(":memory:", &db)); + TF_ASSERT_OK(SetupTensorboardSqliteDb(db.get())); +} + +} // namespace +} // namespace db +} // namespace tensorflow -- GitLab From f3c0fc971a663de8e44121b59ee05a6470887592 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Thu, 26 Oct 2017 18:00:56 -0700 Subject: [PATCH 440/573] Gives Eager Networks unique names, adds related errors. - With no name specified, will create a unique name (consistent with Layer names) - With a name specified, it must be unique Removes containers from Network in favor of the unique name strategy. Does not add any save/restore functionality yet. PiperOrigin-RevId: 173618133 --- tensorflow/contrib/eager/python/network.py | 239 ++++-- .../contrib/eager/python/network_test.py | 681 +++++++++++++++++- tensorflow/python/layers/base.py | 72 +- 3 files changed, 903 insertions(+), 89 deletions(-) diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index 28aed7628e..025d447455 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -19,15 +19,17 @@ from __future__ import division from __future__ import print_function import collections -import uuid - -import six +import weakref from tensorflow.python.estimator import util as estimator_util -from tensorflow.python.framework import ops from tensorflow.python.layers import base from tensorflow.python.ops import variable_scope +# pylint: disable=protected-access +# Explanation for protected-access disable: Network has lots of same-class and +# parent-class references across different objects, and some to private +# functions in base.py which should be reused. + class Network(base.Layer): """Represents the composition of a set of Layers. @@ -35,12 +37,6 @@ class Network(base.Layer): TODO(josh11b,ashankar): - Should "trainable" be changeable on the Network object? - Do we allow add_variable in Network? - - Layer.name and Layer.variables.names are not in sync today - d = tf.layers.Dense(1) - d(tf.constant([[1.]])) - print(d.name) - print(d.variables) - - Note that name provided to __init__ is only for error messages? - Detect layers used in __call__ that weren't registered with track_layer. - Convert inputs to __call__ to tensors. - Prevent variables from being created after the first __call__? @@ -49,9 +45,142 @@ class Network(base.Layer): """ def __init__(self, name=None): + if isinstance(name, variable_scope.VariableScope): + raise ValueError("VariableScopes are not valid Network names.") + if name is not None and "/" in name: + raise ValueError( + "Forward slashes ('/') are not allowed in Network names.") super(Network, self).__init__(name=name) - self._container = uuid.uuid4().hex - self._layers = collections.OrderedDict() + self._layers = [] + self._sub_layer_name_uids = collections.defaultdict(int) + # Initially None, but set to False for networks which are first built as + # top-level. + self._first_parent = None # A weak reference to our first parent. + self._non_network_sublayers = [] + self._owned_layers = {} + # The scope to use if we end up without a parent. + self._default_parent_variable_scope = variable_scope.get_variable_scope() + + def _init_set_name(self, name): + # Anonymous Networks (name=None) defer setting a final name until they are + # (1) added to another Network, or (2) built/called (where (2) is only used + # for a "top level" network). + # + # However, if we were provided an explicit name (name is not None), that + # will always be the final name of the Network; if it turns out not to be + # unique or if variable names can't be prefixed by it we will throw an + # error. + self._name = name + self._base_name = None + + def _finalize_name(self, parent_network): + if not self._name: + if not parent_network: + name_uid_map = base._get_default_graph_uid_map() + else: + name_uid_map = parent_network._sub_layer_name_uids + # Were were not passed a name explicitly (or it was blank), so this is an + # anonymous Network. We make up a unique name. + if parent_network: + avoid_names = parent_network._owned_layers + else: + avoid_names = None + self._name, self._base_name = self._make_unique_name( + name_uid_map=name_uid_map, avoid_names=avoid_names) + if self._first_parent is None or self._first_parent() is None: + # Save a pointer to the parent Network so that we can later check that the + # scope name we get is correct. + if not parent_network: + self._first_parent = parent_network + else: + self._first_parent = weakref.ref(parent_network) + + def _set_scope(self, scope=None): + if self._scope is None: + if not self._first_parent: + first_parent = self._first_parent + else: + first_parent = self._first_parent() + if first_parent is None: + # If we were never added to another Network, or that Network has beed + # garbage collected before being called, then we're a top-level Network. + self._finalize_name( + # Use False to make sure the value sticks and we don't inherit a + # parent if we're added to a network later. + parent_network=False) + if scope is not None: + raise ValueError("Networks may not be created with explicit scopes.") + if first_parent: + first_parent._set_scope() + parent_scope = first_parent._scope + else: + parent_scope = self._default_parent_variable_scope + with variable_scope.variable_scope(parent_scope): + # Make sure variables with this prefix will be unique. + with variable_scope.variable_scope( + None, use_resource=True, default_name=self._name) as scope: + self._scope = scope + scope_name = scope.name + suffix_start = scope_name.rfind("/") + 1 + # rfind is -1 if there is no slash in the string, in which case the + # suffix starts at the beginning of the string (there is no prefix). + scope_suffix = scope_name[suffix_start:] + scope_prefix = scope_name[:suffix_start] + if scope_suffix != self._name: + raise ValueError( + ("A Network named '%s' already exists (or a variable_scope was " + "created with this name). Names must be unique.") % ( + self._name,)) + if (first_parent + and scope_prefix[:-1] != first_parent._scope.name): + raise ValueError( + ("Network variable names must match a nesting of sub-Network " + "names. Expected prefix '%s' from parent network, but got " + "'%s' when attempting to create a variable_scope for Network " + "'%s'. Likely an explicit variable_scope was inserted into " + "the nesting.") % ( + first_parent._scope.name, + scope_prefix[:-1], + self._name)) + elif not first_parent and scope_prefix: + # For the case when this Network is not nested inside any other + # Network, but is in a variable_scope. This is an error for now. + raise ValueError( + "Creating Networks inside named variable_scopes is currently " + "not supported (to ensure that variable names match the names " + "of Networks in which they were first created). To set " + "options, try `with tf.variable_scope(''):`. If this " + "limitation bothers you, please file a feature request.") + for non_network_constituent in self._non_network_sublayers: + if non_network_constituent._scope is None: + if non_network_constituent._first_parent is None: + constituent_first_parent = None + else: + constituent_first_parent = non_network_constituent._first_parent() + if constituent_first_parent: + constituent_first_parent._set_scope() + parent_scope = constituent_first_parent._scope + else: + parent_scope = ( + non_network_constituent._default_parent_variable_scope) + with variable_scope.variable_scope(parent_scope): + # Horrid hack to make Layer variable names which are direct + # sub-layers of Networks conform to the Network variable naming + # conventions. + with variable_scope.variable_scope( + None, use_resource=True, + default_name=non_network_constituent.name) as sub_scope: + non_network_constituent._scope = sub_scope + + @base.Layer.name.getter + def name(self): + if self._name is None: + raise ValueError( + "The network does not yet have a final name, but a name was " + "requested for it. Networks get a name when they are added to " + "another Network via track_layer, or when they are first " + "called/built.") + return self._name def track_layer(self, layer): """Track a Layer in this Network. @@ -76,20 +205,51 @@ class Network(base.Layer): raise TypeError( "Network.track_layer() passed type %s, not a tf.layers.Layer" % (type(layer),)) - if layer.name in self._layers: - if self._layers[layer.name] is layer: - return layer - raise ValueError( - "Attempt to add two Layers with the name '%s' to the same Network " - "'%s'" % (layer.name, self.name)) - self._layers[layer.name] = layer + if isinstance(layer, Network): + layer._finalize_name(parent_network=self) + else: + # `layer` is a non-Network, so it hasn't been named to follow Network + # conventions for contained Layers (i.e. the same conventions as for + # sub-Networks). This renaming is necessary to isolate Network variable + # naming from Layers constructed outside the Network and never added to it + # (because Layers are named globally). + if not layer.built: + if not hasattr(layer, "_first_parent"): + dereferenced_layer_first_parent = None + else: + dereferenced_layer_first_parent = layer._first_parent() + if dereferenced_layer_first_parent is None: + if layer._name != layer._base_name: + # If name and base_name do not match, then this Layer used anonymous + # naming and we have to rename it. Otherwise there's an explicit + # name, and we should respect it (subject to error checking). + layer._name, layer._base_name = layer._make_unique_name( + name_uid_map=self._sub_layer_name_uids, + avoid_names=self._owned_layers) + layer._first_parent = weakref.ref(self) + self._non_network_sublayers.append(layer) + if (not layer.built + and layer._first_parent + and self is layer._first_parent()): + if layer.name in self._owned_layers: + if self._owned_layers[layer.name] is layer: + return layer + raise ValueError( + "Attempt to add two Layers with the name '%s' to the same Network." + % (layer.name)) + self._owned_layers[layer.name] = layer + self._layers.append(layer) return layer def get_layer(self, name=None, index=None): """Get a contained `tf.layers.Layer` either by name or index. Args: - name: String matching one of the names of a contained `Layer`. + name: String matching one of the names of a contained `Layer`. Note that + the names of `Layer`s added to `Network`s may not be unique when doing + layer sharing (i.e. adding a `Layer` to this `Network` which was already + added to another `Network`). The lowest index `Layer` with a matching + name will be returned. index: Integer in [0, number of layers). Layers are assigned an index by the order they are added. @@ -97,19 +257,25 @@ class Network(base.Layer): A `tf.layers.Layer` object. Raises: - ValueError: If neither or both of 'index' or 'name' is specified. + ValueError: If neither or both of 'index' or 'name' is specified, or the + lookup failed. """ if index is not None: if name is not None: raise ValueError("Exactly one of 'index' or 'name' must be provided") if len(self._layers) <= index: - raise ValueError("Was asked to retrieve layer at index " + - str(index) + " but model only has " + str( - len(self._layers)) + " layers.") - return list(self._layers.values())[index] - if name is None: - raise ValueError("Exactly one of 'index' or 'name' must be provided") - return self._layers[index] + raise ValueError("Was asked to retrieve layer at index " + str(index) + + " but model only has " + str(len(self._layers)) + + " layers.") + else: + return self._layers[index] + else: + if not name: + raise ValueError("Provide either a layer name or layer index.") + for layer in self._layers: + if layer.name == name: + return layer + raise ValueError("No such layer: " + name) # The following methods are for implementing the Layer interface. @@ -119,21 +285,21 @@ class Network(base.Layer): # variables in the case of shared layers/variables that appear in # multiple places in the Network? weights = [] - for layer in six.itervalues(self._layers): + for layer in self._layers: weights += layer.weights return weights @property def trainable_weights(self): weights = [] - for layer in six.itervalues(self._layers): + for layer in self._layers: weights += layer.trainable_weights return weights @property def non_trainable_weights(self): weights = [] - for layer in six.itervalues(self._layers): + for layer in self._layers: weights += layer.non_trainable_weights return weights @@ -152,7 +318,7 @@ class Network(base.Layer): @property def layers(self): - return self._layers.values() + return self._layers def add_variable(self, name, shape, dtype=None, initializer=None, regularizer=None, trainable=True, constraint=None): @@ -161,15 +327,6 @@ class Network(base.Layer): "at https://github.com/tensorflow/tensorflow/issues/new if this is " "important to you") - def __call__(self, inputs, *args, **kwargs): - # TODO(josh11b,ashankar,agarwal): Can we reduce the number of context - # managers here and/or move some of the work into the constructor - # for performance reasons? - with ops.container(self._container): - with variable_scope.variable_scope(variable_scope.get_variable_scope(), - use_resource=True): - return super(Network, self).__call__(inputs, *args, **kwargs) - # TODO(josh11b): Support other Layer methods needed for graph mode, such as for # losses and updates diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py index 94cb73ae72..e4cba3f2ed 100644 --- a/tensorflow/contrib/eager/python/network_test.py +++ b/tensorflow/contrib/eager/python/network_test.py @@ -16,19 +16,24 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gc + from tensorflow.contrib.eager.python import network from tensorflow.python.eager import test from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.layers import core from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope # pylint: disable=not-callable class MyNetwork(network.Network): - def __init__(self): - super(MyNetwork, self).__init__(name="abcd") + def __init__(self, name=None): + super(MyNetwork, self).__init__(name=name) self.l1 = self.track_layer(core.Dense(1, use_bias=False)) def call(self, x): @@ -37,6 +42,7 @@ class MyNetwork(network.Network): class NetworkTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testTrainableAttribute(self): net = network.Network() self.assertTrue(net.trainable) @@ -44,41 +50,676 @@ class NetworkTest(test.TestCase): net.trainable = False self.assertTrue(net.trainable) + @test_util.run_in_graph_and_eager_modes() def testNetworkCall(self): - net = MyNetwork() + net = MyNetwork(name="abcd") net(constant_op.constant([[2.0]])) # Force variables to be created. self.assertEqual(1, len(net.trainable_variables)) - net.trainable_variables[0].assign([[17.0]]) + self.evaluate(net.trainable_variables[0].assign([[17.0]])) # TODO(josh11b): Support passing Python values to networks. result = net(constant_op.constant([[2.0]])) - self.assertEqual(34.0, result.numpy()) + self.assertEqual(34.0, self.evaluate(result)) + + def testNoReferenceCyclesAfterCall(self): + + class ChildNetwork(network.Network): + + def __init__(self, name=None): + super(ChildNetwork, self).__init__(name=name) + + def call(self, x): + return x * 2. + + class ParentNetwork(network.Network): + + def __init__(self, name=None): + super(ParentNetwork, self).__init__(name=name) + self.l1 = self.track_layer(ChildNetwork()) + + def call(self, x): + return self.l1(x) + + one = constant_op.constant([[1.0]]) + gc.disable() + gc.collect() + previous_gc_debug_flags = gc.get_debug() + gc.set_debug(gc.DEBUG_SAVEALL) + preexisting = len(gc.garbage) + net = ParentNetwork() + net(one) + del net + gc.collect() + # There should be no additional garbage requiring collection. + self.assertEqual(preexisting, len(gc.garbage)) + gc.set_debug(previous_gc_debug_flags) + gc.enable() + + @test_util.run_in_graph_and_eager_modes() + def testAnonymousNoNameInitially(self): + net = MyNetwork() + with self.assertRaisesRegexp(ValueError, "does not yet have a final name"): + net.name # pylint: disable=pointless-statement + + @test_util.run_in_graph_and_eager_modes() + def testExplicitHasNameInitially(self): + net = MyNetwork(name="abcd") + self.assertEqual("abcd", net.name) + + @test_util.run_in_graph_and_eager_modes() + def testUsingResourceVariables(self): + net = MyNetwork() + net(constant_op.constant([[0.]])) + self.assertIsInstance(net.trainable_weights[0], + resource_variable_ops.ResourceVariable) + + @test_util.run_in_graph_and_eager_modes() + def testDuplicateNameError(self): + one = constant_op.constant([[1.]]) + net = MyNetwork(name="foo") + net(one) + with self.assertRaisesRegexp( + ValueError, "named 'foo' already exists"): + net1 = MyNetwork(name="foo") + net1(one) + + @test_util.run_in_graph_and_eager_modes() + def testWrappingInVariableScope(self): + with variable_scope.variable_scope("outside_scope"): + net = MyNetwork() + one = constant_op.constant([[1.]]) + with self.assertRaisesRegexp( + ValueError, + ("Creating Networks inside named variable_scopes is currently not " + "supported")): + net(one) + # Alternatively, we could re-name the Network to match the variable_scope: + # self.assertEqual("outside_scope/my_network_1", net.name) + # self.assertStartsWith( + # expected_start="outside_scope/my_network_1/dense/", + # actual=net.trainable_weights[0].name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerNamesRespected(self): + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer( + core.Dense(1, use_bias=False, name="explicit_name")) + + def call(self, x): + return self.first(x) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith(expected_start="parent_network_1/explicit_name/", + actual=net.trainable_weights[0].name) + self.assertEqual("explicit_name", net.first.name) + + @test_util.run_in_graph_and_eager_modes() + def testWrappingInAnonymousVariableScope(self): + # Named outside variable_scopes are not supported at the moment. However, + # blank-named top level variable scopes do not change variable names, and so + # can be used to set the properties of Network variables. + was_called = [False] + def _custom_getter(getter, *args, **kwargs): + was_called[0] = True + return getter(*args, **kwargs) + with variable_scope.variable_scope("", custom_getter=_custom_getter): + net = MyNetwork() + one = constant_op.constant([[1.]]) + net(one) + self.assertTrue(was_called[0]) - def testNetworkAsAGraph(self): - self.skipTest("TODO(ashankar,josh11b): FIX THIS") - # Verify that we're using ResourceVariables + @test_util.run_in_graph_and_eager_modes() + def testReasonableSlashError(self): + with self.assertRaisesRegexp( + ValueError, "not allowed in Network names"): + MyNetwork(name="slash/slash") + @test_util.run_in_graph_and_eager_modes() + def testNoVariableScopeNames(self): + with self.assertRaisesRegexp( + ValueError, "VariableScopes are not valid Network names"): + with variable_scope.variable_scope("some_scope") as vs: + MyNetwork(name=vs) + + @test_util.run_in_graph_and_eager_modes() + def testVariableScopeNameCollision(self): + with variable_scope.variable_scope("abcd"): + pass + with self.assertRaisesRegexp( + ValueError, "or a variable_scope was created with this name"): + net = MyNetwork(name="abcd") + one = constant_op.constant([[1.]]) + net(one) + + @test_util.run_in_graph_and_eager_modes() def testNetworkVariablesDoNotInterfere(self): - self.skipTest("TODO: FIX THIS") + core.Dense(1, use_bias=True) # Should not interfere with naming. net1 = MyNetwork() net2 = MyNetwork() + one = constant_op.constant([[1.]]) + net1(one) + net2(one) + # Layer names typically are globally unique rather than being unique within + # the scope of their first use. However, within a Network they must be named + # locally so that previous Layer consutrciton does not interfere with + # variable naming (e.g. add a Layer construction before the Network, + # suddenly your previously saved checkpoint is incompatible). + self.assertEqual("dense_1", net1.l1.name) + self.assertEqual("dense_1", net2.l1.name) + self.evaluate(net1.trainable_weights[0].assign([[1.]])) + self.evaluate(net2.trainable_weights[0].assign([[2.]])) + self.assertEqual(2., self.evaluate(net2.trainable_weights[0])) + self.assertEqual(1., self.evaluate(net1.trainable_weights[0])) + self.assertStartsWith(expected_start="my_network_1/dense_1/", + actual=net1.trainable_weights[0].name) + self.assertStartsWith(expected_start="my_network_2/dense_1/", + actual=net2.trainable_weights[0].name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableAnonymous(self): + + # The case where no explicit names are specified. We make up unique names, + # and these should match the variable names. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense", + actual=net.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense", + actual=net.first.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_1/my_network_2/dense", + actual=net.trainable_weights[1].name) + self.assertStartsWith(expected_start="parent_network_1/my_network_2/dense", + actual=net.second.trainable_weights[0].name) + self.assertEqual("parent_network_1", net.name) + self.assertEqual("my_network_1", net.first.name) + self.assertEqual("my_network_2", net.second.name) - print(type(net1(one))) + net2 = ParentNetwork() net2(one) + self.assertStartsWith(expected_start="parent_network_2/my_network_1/dense", + actual=net2.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_2/my_network_1/dense", + actual=net2.first.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_2/my_network_2/dense", + actual=net2.trainable_weights[1].name) + self.assertStartsWith(expected_start="parent_network_2/my_network_2/dense", + actual=net2.second.trainable_weights[0].name) + self.assertEqual("parent_network_2", net2.name) + self.assertEqual("my_network_1", net2.first.name) + self.assertEqual("my_network_2", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicit(self): + + # We have explicit network names and everything is globally unique. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__(name="unique_parent_name") + self.first = self.track_layer( + MyNetwork(name="first_unique_child_name")) + self.second = self.track_layer( + MyNetwork(name="second_unique_child_name")) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith( + expected_start="unique_parent_name/first_unique_child_name/dense", + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="unique_parent_name/second_unique_child_name/dense", + actual=net.trainable_weights[1].name) + self.assertEqual("unique_parent_name", net.name) + self.assertEqual("first_unique_child_name", net.first.name) + self.assertEqual("second_unique_child_name", net.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerNetworkNameInteractions(self): + + # Same base name as core.Dense; Networks and non-Network Layers with the + # same base name should use the same numbering system. + class Dense(network.Network): + + def __init__(self): + super(Dense, self).__init__() + self.first = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.first(x) + + class MixedLayerNetwork(network.Network): + + def __init__(self): + super(MixedLayerNetwork, self).__init__() + self.first = self.track_layer(core.Dense(1, use_bias=False)) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + self.third = self.track_layer(Dense()) + self.fourth = self.track_layer(core.Dense(1, use_bias=False)) + self.fifth = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.fifth(self.fourth(self.third(self.second(self.first(x))))) + + one = constant_op.constant([[1.]]) + net = MixedLayerNetwork() + net(one) + self.assertEqual("dense_1", net.first.name) + self.assertEqual("dense_2", net.second.name) + self.assertEqual("dense_3", net.third.name) + self.assertEqual("dense_4", net.fourth.name) + self.assertEqual("dense_5", net.fifth.name) + # Note that this is _not_ the default naming behavior for Layers. Layers + # which are added to Networks follow Network variable naming conventions + # (i.e. variable names = network name unless variable sharing). Nested + # Layers revert to Layer behavior. + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_1/", + actual=net.trainable_weights[0].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_2/", + actual=net.trainable_weights[1].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_3/", + actual=net.trainable_weights[2].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_4/", + actual=net.trainable_weights[3].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_5/", + actual=net.trainable_weights[4].name) + self.assertEqual("mixed_layer_network_1", net.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicitCollisions(self): + + # We have explicit network names and they are unique within the layer + # they're added to. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__(name="nonunique_name") + self.first = self.track_layer( + MyNetwork(name="nonunique_name")) + self.second = self.track_layer( + MyNetwork(name="second_unique_child_name")) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith( + expected_start="nonunique_name/nonunique_name/dense", + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="nonunique_name/second_unique_child_name/dense", + actual=net.trainable_weights[1].name) + self.assertEqual("nonunique_name", net.name) + self.assertEqual("nonunique_name", net.first.name) + self.assertEqual("second_unique_child_name", net.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicitWithAnonymousParent(self): + + # A parent network is instantiated multiple times with explicitly named + # children. We shouldn't throw any name errors. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer( + MyNetwork(name="first_unique_child_name")) + self.second = self.track_layer( + MyNetwork(name="second_unique_child_name")) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith( + expected_start="parent_network_1/first_unique_child_name/dense_1/", + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="parent_network_1/second_unique_child_name/dense_1/", + actual=net.trainable_weights[1].name) + self.assertEqual("parent_network_1", net.name) + self.assertEqual("first_unique_child_name", net.first.name) + self.assertEqual("second_unique_child_name", net.second.name) + + net2 = ParentNetwork() + net2(one) + self.assertStartsWith( + expected_start="parent_network_2/first_unique_child_name/dense", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="parent_network_2/second_unique_child_name/dense", + actual=net2.trainable_weights[1].name) + self.assertEqual("parent_network_2", net2.name) + self.assertEqual("first_unique_child_name", net2.first.name) + self.assertEqual("second_unique_child_name", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicitSameLayerCollisions(self): + + # We have explicit network names and they are _not_ unique within the layer + # they're added to. Error. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__(name="unique_parent_name") + self.first = self.track_layer(MyNetwork(name="nonunique_name")) + self.second = self.track_layer(MyNetwork(name="nonunique_name")) + + def call(self, x): + return self.second(self.first(x)) + + with self.assertRaisesRegexp(ValueError, "nonunique_name"): + ParentNetwork() + + @test_util.run_in_graph_and_eager_modes() + def testAnonymousVariableSharing(self): + + # Two "owned" Networks + class FirstParentNetwork(network.Network): + + def __init__(self): + super(FirstParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = FirstParentNetwork() + net(one) + + # One Network shared with FirstParentNetwork, one owned Network. Same name, + # but this is OK because only one is owned. This name collision is + # avoidable; we could have looked at the base_name of the non-owned Network + # and incremented our naming based on that. + class SecondParentNetwork(network.Network): + + def __init__(self): + super(SecondParentNetwork, self).__init__() + self.first = self.track_layer(net.first) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + net2 = SecondParentNetwork() + net2(one) + + self.assertStartsWith( + expected_start="first_parent_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="second_parent_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[1].name) + self.assertEqual("second_parent_network_1", net2.name) + self.assertTrue(net2.first is net.first) + self.assertEqual("my_network_1", net2.first.name) + self.assertEqual("my_network_1", net2.second.name) + + # No name collision; the owned Network is added first and has a different + # name than the shared Network. + class ThirdParentNetwork(network.Network): + + def __init__(self): + super(ThirdParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(net.second) + + def call(self, x): + return self.second(self.first(x)) + + net3 = ThirdParentNetwork() + net3(one) + + self.assertStartsWith( + expected_start="third_parent_network_1/my_network_1/dense", + actual=net3.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_parent_network_1/my_network_2/dense", + actual=net3.trainable_weights[1].name) + self.assertEqual("third_parent_network_1", net3.name) + self.assertTrue(net3.second is net.second) + self.assertEqual("my_network_1", net3.first.name) + self.assertEqual("my_network_2", net3.second.name) + + # "Unavoidable" same-name Layer. The owned name is added first (fixed), then + # a shared Network is added with the same name. + class FourthParentNetwork(network.Network): + + def __init__(self): + super(FourthParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(net.first) + + def call(self, x): + return self.second(self.first(x)) + + net4 = FourthParentNetwork() + net4(one) + + self.assertStartsWith( + expected_start="fourth_parent_network_1/my_network_1/dense_1/", + actual=net4.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_parent_network_1/my_network_1/dense_1/", + actual=net4.trainable_weights[1].name) + self.assertEqual("fourth_parent_network_1", net4.name) + self.assertTrue(net4.second is net.first) + self.assertEqual("my_network_1", net4.first.name) + self.assertEqual("my_network_1", net4.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testRecursiveLayerRenaming(self): + core.Dense(1) # Under default Layer naming, would change subsequent names. + + class NetworkWithLayerChildren(network.Network): + + def __init__(self): + super(NetworkWithLayerChildren, self).__init__() + self.first = self.track_layer(core.Dense(1, use_bias=False)) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer(NetworkWithLayerChildren()) + self.second = self.track_layer(NetworkWithLayerChildren()) + + def call(self, x): + return self.second(self.first(x)) + + net = ParentNetwork() + one = constant_op.constant([[1.]]) + net(one) + + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_1/" + "dense_1/"), + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_1/" + "dense_2/"), + actual=net.trainable_weights[1].name) + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_2/" + "dense_1/"), + actual=net.trainable_weights[2].name) + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_2/" + "dense_2/"), + actual=net.trainable_weights[3].name) + self.assertEqual("parent_network_1", net.name) + self.assertEqual("network_with_layer_children_1", net.first.name) + self.assertEqual("network_with_layer_children_2", net.second.name) + self.assertEqual("dense_1", net.first.first.name) + self.assertEqual("dense_2", net.first.second.name) + self.assertEqual("dense_1", net.second.first.name) + self.assertEqual("dense_2", net.second.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testCallInDifferentOrderThanConstruct(self): + shared_network = MyNetwork() + + class FirstNetwork(network.Network): + + def __init__(self): + super(FirstNetwork, self).__init__() + self.first = self.track_layer(shared_network) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + class SecondNetwork(network.Network): + + def __init__(self): + super(SecondNetwork, self).__init__() + self.first = self.track_layer(shared_network) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + net1 = FirstNetwork() + net2 = SecondNetwork() + + one = constant_op.constant([[1.]]) + net2(one) + net1(one) + + self.assertStartsWith( + expected_start="first_network_1/my_network_1/dense_1/", + actual=net1.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_network_1/my_network_2/dense_1/", + actual=net1.trainable_weights[1].name) + self.assertStartsWith( + expected_start="first_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="second_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[1].name) + self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0]) + self.assertEqual("first_network_1", net1.name) + self.assertEqual("my_network_1", net1.first.name) + self.assertEqual("my_network_2", net1.second.name) + self.assertTrue(net2.first is net1.first) + self.assertEqual("my_network_1", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerCallInDifferentOrderThanConstruct(self): + # Same idea as testCallInDifferentOrderThanConstruct, but this time with a + # non-Network Layer shared between two Networks rather than a + # Network. Naming should follow the same rules. + shared_layer = core.Dense(1, use_bias=False) + + class FirstNetwork(network.Network): + + def __init__(self): + super(FirstNetwork, self).__init__() + self.first = self.track_layer(shared_layer) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + class SecondNetwork(network.Network): + + def __init__(self): + super(SecondNetwork, self).__init__() + self.first = self.track_layer(shared_layer) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + net1 = FirstNetwork() + net2 = SecondNetwork() + + one = constant_op.constant([[1.]]) + net2(one) + net1(one) + + self.assertStartsWith( + expected_start="first_network_1/dense_1/", + actual=net1.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_network_1/dense_2/", + actual=net1.trainable_weights[1].name) + self.assertStartsWith( + expected_start="first_network_1/dense_1/", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="second_network_1/dense_1/", + actual=net2.trainable_weights[1].name) + self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0]) + self.assertEqual("first_network_1", net1.name) + self.assertEqual("dense_1", net1.first.name) + self.assertEqual("dense_2", net1.second.name) + self.assertTrue(net2.first is net1.first) + self.assertEqual("dense_1", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerAlreadyBuilt(self): + one = constant_op.constant([[1.]]) + core.Dense(1, use_bias=False) # pre-built layers use global naming + one = constant_op.constant([[1.]]) + core.Dense(1, use_bias=False)(one) + shared_layer = core.Dense(1, use_bias=False) + shared_layer(one) + + class FirstNetwork(network.Network): + + def __init__(self): + super(FirstNetwork, self).__init__() + self.first = self.track_layer(shared_layer) + self.second = self.track_layer(core.Dense(1, use_bias=False)) - net1.trainable_weights[0].assign(constant_op.constant([[1.]])) - net2.trainable_weights[0].assign(constant_op.constant([[2.]])) + def call(self, x): + return self.second(self.first(x)) - print("NET1") - print(net1.name) - print(net1.variables) - print(net1(one)) + net = FirstNetwork() + net(one) - print("NET2") - print(net2.name) - print(net2.variables) - print(net2(one)) + self.assertStartsWith( + expected_start="dense_1/", # Pre-built layers have variable names which + # do not match their layer names. + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_network_1/dense_1/", + actual=net.trainable_weights[1].name) + self.assertTrue( + net.trainable_weights[0] is shared_layer.trainable_weights[0]) + self.assertEqual("first_network_1", net.name) + self.assertEqual("dense_3", net.first.name) + self.assertEqual("dense_1", net.second.name) class SequentialTest(test.TestCase): diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index 134d4fc8e2..8c2ee1f103 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -122,16 +122,7 @@ class Layer(object): self._inbound_nodes = [] self._outbound_nodes = [] - # Determine layer name (non-unique). - if isinstance(name, vs.VariableScope): - base_name = name.name - else: - base_name = name - self._name = name - if not name: - base_name = _to_snake_case(self.__class__.__name__) - self._name = _unique_layer_name(base_name) - self._base_name = base_name + self._init_set_name(name) # Determine variable scope. scope = kwargs.get('_scope') @@ -147,6 +138,17 @@ class Layer(object): batch_size = kwargs.get('batch_size') self._batch_input_shape = (batch_size,) + tuple(kwargs['input_shape']) + def _init_set_name(self, name): + # Determine layer name (non-unique). + if isinstance(name, vs.VariableScope): + base_name = name.name + else: + base_name = name + self._name = name + if not name: + self._name, base_name = self._make_unique_name() + self._base_name = base_name + @property def dtype(self): return self._dtype @@ -399,6 +401,12 @@ class Layer(object): """ return input_shape + def _make_unique_name(self, name_uid_map=None, avoid_names=None): + base_name = _to_snake_case(self.__class__.__name__) + name = _unique_layer_name(base_name, name_uid_map=name_uid_map, + avoid_names=avoid_names) + return (name, base_name) + def _set_scope(self, scope=None): if self._scope is None: # If constructed with _scope=None, lazy setting of scope. @@ -1507,19 +1515,11 @@ class Network(Layer): # TODO(fchollet): check that all inputs and outputs are DeferredTensors. pass - # Set layer name and scope - if isinstance(name, vs.VariableScope): - base_name = name.name - else: - base_name = name - self._name = name - if not name: - base_name = _to_snake_case(self.__class__.__name__) - self._name = _unique_layer_name(base_name) + self._init_set_name(name) self._activity_regularizer = None - with vs.variable_scope(None, default_name=base_name) as captured_scope: + with vs.variable_scope( + None, default_name=self._base_name) as captured_scope: self._scope = captured_scope - self._base_name = base_name call_fn_args = estimator_util.fn_args(self.call) self._compute_previous_mask = ('mask' in call_fn_args or hasattr(self, 'compute_mask')) @@ -2354,11 +2354,24 @@ def _collect_previous_mask(input_tensors): PER_GRAPH_LAYER_NAME_UIDS = weakref.WeakKeyDictionary() -def _unique_layer_name(name): +def _get_default_graph_uid_map(): + graph = ops.get_default_graph() + name_uid_map = PER_GRAPH_LAYER_NAME_UIDS.get(graph, None) + if name_uid_map is None: + name_uid_map = collections.defaultdict(int) + PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map + return name_uid_map + + +def _unique_layer_name(name, name_uid_map=None, avoid_names=None): """Makes a layer name (or arbitrary string) unique within a TensorFlow graph. Arguments: name: String name to make unique. + name_uid_map: An optional defaultdict(int) to use when creating unique + names. If None (default), uses a per-Graph dictionary. + avoid_names: An optional set or dict with names which should not be used. If + None (default) does not avoid any names. Returns: Unique string name. @@ -2370,9 +2383,12 @@ def _unique_layer_name(name): _unique_layer_name('dense') # dense_2 ``` """ - graph = ops.get_default_graph() - if graph not in PER_GRAPH_LAYER_NAME_UIDS: - PER_GRAPH_LAYER_NAME_UIDS[graph] = collections.defaultdict(int) - layer_name_uids = PER_GRAPH_LAYER_NAME_UIDS[graph] - layer_name_uids[name] += 1 - return name + '_' + str(layer_name_uids[name]) + if name_uid_map is None: + name_uid_map = _get_default_graph_uid_map() + if avoid_names is None: + avoid_names = set() + proposed_name = None + while proposed_name is None or proposed_name in avoid_names: + name_uid_map[name] += 1 + proposed_name = name + '_' + str(name_uid_map[name]) + return proposed_name -- GitLab From abebb5f3fa6799e4fc1f2de1156a7c968c8473b8 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Thu, 26 Oct 2017 18:26:28 -0700 Subject: [PATCH 441/573] TFE: Add compatibility doc string to Saver and related functions Also change `ValueError`s to `RuntimeError`s to be consistent with other errors of this kind. PiperOrigin-RevId: 173620243 --- tensorflow/python/training/input.py | 8 ++++- tensorflow/python/training/optimizer.py | 12 +++++--- tensorflow/python/training/saver.py | 41 +++++++++++++++++++------ 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index b999dbedb6..e7adbf11b4 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -146,9 +146,15 @@ def input_producer(input_tensor, Raises: ValueError: If the shape of the input cannot be inferred from the arguments. + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Queue-using input pipelines are not supported when eager execution is enabled. + Please use tf.data to ingest data into your model instead. + @end_compatibility """ if context.in_eager_mode(): - raise ValueError( + raise RuntimeError( "Queue-using input pipelines are not supported when eager execution is" " enabled. Please use tf.data to ingest data into your model instead.") with ops.name_scope(name, "input_producer", [input_tensor]): diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index d6ca52cd1b..915214dbfa 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -397,6 +397,8 @@ class Optimizer(object): Raises: TypeError: If `var_list` contains anything else than `Variable` objects. ValueError: If some arguments are invalid. + RuntimeError: If called with eager execution enabled and if `grad_loss` + is not `None` or `loss` is not callable. @compatibility(eager) When eager execution is enabled, `loss` should be a Python function that @@ -411,11 +413,13 @@ class Optimizer(object): """ if context.in_eager_mode(): if grad_loss is not None: - raise ValueError("`grad_loss` argument to Optimizer.compute_gradients " - "not supported when eager execution is enabled.") + raise RuntimeError( + "`grad_loss` argument to Optimizer.compute_gradients " + "not supported when eager execution is enabled.") if not callable(loss): - raise ValueError("`loss` passed to Optimizer.compute_gradients should " - "be a function when eager execution is enabled.") + raise RuntimeError( + "`loss` passed to Optimizer.compute_gradients should " + "be a function when eager execution is enabled.") # TODO(agarwal): consider passing parameters to the `loss` function. if var_list is None: return backprop.implicit_grad(loss)() diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 145b44e2e0..9d784b2745 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -1198,15 +1198,22 @@ class Saver(object): Raises: TypeError: If `var_list` is invalid. ValueError: If any of the keys or values in `var_list` are not unique. + RuntimeError: If eager execution is enabled and`var_list` does not specify + a list of varialbes to save. + + @compatibility(eager) + When eager execution is enabled, `var_list` must specify a `list` or `dict` + of variables to save. Otherwise, a `RuntimeError` will be raised. + @end_compatibility """ if defer_build and var_list: raise ValueError( "If `var_list` is provided then build cannot be deferred. " "Either set defer_build=False or var_list=None.") if context.in_eager_mode() and var_list is None: - raise ValueError( - "When eager execution is enabled, `var_list` must specify a list of " - "variables to save") + raise RuntimeError( + "When eager execution is enabled, `var_list` must specify a list or " + "dict of variables to save") self._var_list = var_list self._reshape = reshape self._sharded = sharded @@ -1231,7 +1238,7 @@ class Saver(object): def build(self): if context.in_eager_mode(): - raise ValueError("Use save/restore instead of build in eager mode.") + raise RuntimeError("Use save/restore instead of build in eager mode.") self._build(self._filename, build_save=True, build_restore=True) def _build_eager(self, checkpoint_path, build_save, build_restore): @@ -1802,11 +1809,19 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False, A None value is returned if no variables exist in the `MetaGraphDef` (i.e., there are no variables to restore). + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Exporting/importing meta graphs is not supported. No graph exists when eager + execution is enabled. + @end_compatibility """ # pylint: disable=g-doc-exception if context.in_eager_mode(): - raise ValueError("Exporting/importing meta graphs is not supported when " - "eager execution is enabled. No graph exists when eager " - "execution is enabled.") + raise RuntimeError("Exporting/importing meta graphs is not supported when " + "eager execution is enabled. No graph exists when eager " + "execution is enabled.") if not isinstance(meta_graph_or_file, meta_graph_pb2.MetaGraphDef): meta_graph_def = meta_graph.read_meta_graph_file(meta_graph_or_file) else: @@ -1872,11 +1887,17 @@ def export_meta_graph(filename=None, Raises: ValueError: When the `GraphDef` is larger than 2GB. + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Exporting/importing meta graphs is not supported. No graph exists when eager + execution is enabled. + @end_compatibility """ if context.in_eager_mode(): - raise ValueError("Exporting/importing meta graphs is not supported when " - "eager execution is enabled. No graph exists when eager " - "execution is enabled.") + raise RuntimeError("Exporting/importing meta graphs is not supported when " + "eager execution is enabled. No graph exists when eager " + "execution is enabled.") meta_graph_def, _ = meta_graph.export_scoped_meta_graph( filename=filename, meta_info_def=meta_info_def, -- GitLab From b113d082ac6320adaaa0205cd77ab815ff40bc16 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 26 Oct 2017 19:06:43 -0700 Subject: [PATCH 442/573] Exclude 'self' from function arguments returned by util.fn_args for callables and bounded methods. PiperOrigin-RevId: 173622989 --- .../estimator/python/estimator/extenders.py | 5 +-- .../contrib/tpu/python/tpu/tpu_estimator.py | 2 - tensorflow/python/estimator/estimator.py | 4 -- tensorflow/python/estimator/util.py | 39 ++++++++++--------- tensorflow/python/estimator/util_test.py | 11 +++++- 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/estimator/python/estimator/extenders.py b/tensorflow/contrib/estimator/python/estimator/extenders.py index 3e5eb3390f..29c3c73585 100644 --- a/tensorflow/contrib/estimator/python/estimator/extenders.py +++ b/tensorflow/contrib/estimator/python/estimator/extenders.py @@ -27,7 +27,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.ops import clip_ops from tensorflow.python.training import optimizer as optimizer_lib -from tensorflow.python.util import tf_inspect + _VALID_METRIC_FN_ARGS = set(['features', 'labels', 'predictions', 'config']) @@ -317,9 +317,6 @@ class _TransformGradients(optimizer_lib.Optimizer): def _verify_metric_fn_args(metric_fn): args = set(estimator_util.fn_args(metric_fn)) - if tf_inspect.ismethod(metric_fn): - if 'self' in args: - args.remove('self') invalid_args = list(args - _VALID_METRIC_FN_ARGS) if invalid_args: raise ValueError('metric_fn (%s) has following not expected args: %s' % diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 805de16468..5a3b831429 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -1106,8 +1106,6 @@ class _EvalMetrics(object): if isinstance(eval_metrics[1], (tuple, list)): fn_args = util.fn_args(eval_metrics[0]) - if 'self' in fn_args: - fn_args = tuple([arg for arg in fn_args if arg != 'self']) if len(eval_metrics[1]) != len(fn_args): raise RuntimeError( 'In TPUEstimatorSpec.eval_metrics, length of tensors does not ' diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 2a4d77b1a6..f198b051cf 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -52,7 +52,6 @@ from tensorflow.python.training import training from tensorflow.python.training import training_util from tensorflow.python.util import compat from tensorflow.python.util import nest -from tensorflow.python.util import tf_inspect _VALID_MODEL_FN_ARGS = set( @@ -925,9 +924,6 @@ def _verify_model_fn_args(model_fn, params): logging.warning('Estimator\'s model_fn (%s) includes params ' 'argument, but params are not passed to Estimator.', model_fn) - if tf_inspect.ismethod(model_fn): - if 'self' in args: - args.remove('self') non_valid_args = list(args - _VALID_MODEL_FN_ARGS) if non_valid_args: raise ValueError('model_fn (%s) has following not expected args: %s' % diff --git a/tensorflow/python/estimator/util.py b/tensorflow/python/estimator/util.py index de35e66bdf..12f2592d84 100644 --- a/tensorflow/python/estimator/util.py +++ b/tensorflow/python/estimator/util.py @@ -19,10 +19,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_inspect +def _is_bounded_method(fn): + return tf_inspect.ismethod(fn) and (fn.__self__ is not None) + + +def _is_callable_object(obj): + return hasattr(obj, '__call__') and tf_inspect.ismethod(obj.__call__) + + def fn_args(fn): """Get argument names for function-like object. @@ -36,22 +46,13 @@ def fn_args(fn): ValueError: if partial function has positionally bound arguments """ _, fn = tf_decorator.unwrap(fn) - - # Handle callables. - if hasattr(fn, '__call__') and tf_inspect.ismethod(fn.__call__): - return tuple(tf_inspect.getargspec(fn.__call__).args) - - # Handle functools.partial and similar objects. - if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'): - # Handle nested partial. - original_args = fn_args(fn.func) - if not original_args: - return tuple() - - return tuple([ - arg for arg in original_args[len(fn.args):] - if arg not in set((fn.keywords or {}).keys()) - ]) - - # Handle function. - return tuple(tf_inspect.getargspec(fn).args) + if isinstance(fn, functools.partial): + args = fn_args(fn.func) + args = [a for a in args[len(fn.args):] if a not in (fn.keywords or [])] + else: + if _is_callable_object(fn): + fn = fn.__call__ + args = tf_inspect.getargspec(fn).args + if _is_bounded_method(fn): + args.remove('self') + return tuple(args) diff --git a/tensorflow/python/estimator/util_test.py b/tensorflow/python/estimator/util_test.py index 3f8122c407..4b2c8d7637 100644 --- a/tensorflow/python/estimator/util_test.py +++ b/tensorflow/python/estimator/util_test.py @@ -38,7 +38,16 @@ class FnArgsTest(test.TestCase): def __call__(self, a, b): return a + b - self.assertEqual(('self', 'a', 'b'), util.fn_args(Foo())) + self.assertEqual(('a', 'b'), util.fn_args(Foo())) + + def test_bounded_method(self): + + class Foo(object): + + def bar(self, a, b): + return a + b + + self.assertEqual(('a', 'b'), util.fn_args(Foo().bar)) def test_partial_function(self): expected_test_arg = 123 -- GitLab From a710cb323a69458ccda772a65bb20433419dd1d9 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Thu, 26 Oct 2017 19:58:54 -0700 Subject: [PATCH 443/573] Internal change. PiperOrigin-RevId: 173626040 --- tensorflow/contrib/eager/python/saver.py | 56 ++++++++++++++---------- tensorflow/python/training/saver.py | 15 +++---- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py index 404f77105a..d74e0fef3e 100644 --- a/tensorflow/contrib/eager/python/saver.py +++ b/tensorflow/contrib/eager/python/saver.py @@ -90,8 +90,8 @@ def restore_variables_on_create(save_path, map_func=None): for k, _ in checkpoint_utils.list_variables(save_path): ckpt_var_cache[k] = reader.get_tensor(k) - old_init = getattr( - resource_variable_ops.ResourceVariable, "_init_from_args", None) + old_init = getattr(resource_variable_ops.ResourceVariable, + "_init_from_args", None) assert old_init, "ResourceVariable misses _init_from_args method." setattr(resource_variable_ops.ResourceVariable, "_init_from_args", _init_from_checkpoint) @@ -114,42 +114,54 @@ def restore_variables_on_create(save_path, map_func=None): class Saver(object): - """A simple tf.train.Saver adapter for eager mode. - - save and restore API are similar to the tf.train.Saver, except that - session is not needed. - - Args: - var_list: Same as tf.train.Saver. + """A tf.train.Saver adapter for use when eager execution is enabled. """ def __init__(self, var_list): + """A tf.train.Saver adapter for use when eager execution is enabled. + + The API, and on-disk format, mimic tf.train.Saver except that no + Session is needed. + + Args: + var_list: The list of variables that will be saved and restored. Either a + list of `tfe.Variable` objects, or a dictionary mapping names to + `tfe.Variable` objects. + + Raises: + RuntimeError: if invoked when eager execution has not been enabled. + """ if context.in_graph_mode(): - raise ValueError("Currently, tfe.Saver can only be used when eager " - "execution is enabled. Use tf.train.Saver when " - "building graphs.") + raise RuntimeError("tfe.Saver can only be used when eager " + "execution is enabled. Use tf.train.Saver when " + "building graphs.") self._saver = _saver.Saver(var_list=var_list) - def save(self, save_path, global_step=None): + def save(self, file_prefix, global_step=None): """Saves variables. Args: - save_path: See save method in tf.train.Saver. - global_step: See save method in tf.train.Saver. + file_prefix: Path prefix of files created for the checkpoint. + global_step: If provided the global step number is appended to file_prefix + to create the checkpoint filename. The optional argument can be a + Tensor, a Variable, or an integer. Returns: - See save method in tf.train.Saver. + A string: prefix of filenames created for the checkpoint. This may be + an extension of file_prefix that is suitable to pass as an argument + to a subsequent call to `restore()`. """ with ops.device("/device:CPU:0"): - return self._saver.save(None, save_path, write_meta_graph=False, - global_step=global_step) + return self._saver.save( + None, file_prefix, write_meta_graph=False, global_step=global_step) - def restore(self, save_path): + def restore(self, file_prefix): """Restores previously saved variables. Args: - save_path: See restore method in tf.train.Saver. + file_prefix: Path prefix where parameters were previously saved. + Typically obtained from a previous `save()` call, or from + @{tf.train.latest_checkpoint}. """ with ops.device("/device:CPU:0"): - self._saver.restore(None, save_path) - + self._saver.restore(None, file_prefix) diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 9d784b2745..60420eb86a 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -1510,18 +1510,17 @@ class Saver(object): It requires a session in which the graph was launched. The variables to save must also have been initialized. - The method returns the path of the newly created checkpoint file. This - path can be passed directly to a call to `restore()`. + The method returns the path prefix of the newly created checkpoint files. + This string can be passed directly to a call to `restore()`. Args: - sess: A Session to use to save the variables. None in eager mode. - save_path: String. Path to the checkpoint filename. If the saver is - `sharded`, this is the prefix of the sharded checkpoint filename. + sess: A Session to use to save the variables. + save_path: String. Prefix of filenames created for the checkpoint. global_step: If provided the global step number is appended to - `save_path` to create the checkpoint filename. The optional argument + `save_path` to create the checkpoint filenames. The optional argument can be a `Tensor`, a `Tensor` name or an integer. latest_filename: Optional name for the protocol buffer file that will - contains the list of most recent checkpoint filenames. That file, + contains the list of most recent checkpoints. That file, kept in the same directory as the checkpoint files, is automatically managed by the saver to keep track of recent checkpoints. Defaults to 'checkpoint'. @@ -1532,7 +1531,7 @@ class Saver(object): `CheckpointStateProto`. Returns: - A string: path at which the variables were saved. If the saver is + A string: path prefix used for the checkpoint files. If the saver is sharded, this string ends with: '-?????-of-nnnnn' where 'nnnnn' is the number of shards created. If the saver is empty, returns None. -- GitLab From e7e312a11fdb2d6c3c8dc183b6fb2d2e55b43242 Mon Sep 17 00:00:00 2001 From: Rohan Varma Date: Thu, 26 Oct 2017 20:58:13 -0700 Subject: [PATCH 444/573] reuse=False --- tensorflow/python/ops/variable_scope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 08be8574f3..91c53f401b 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -581,7 +581,7 @@ class _VariableStore(object): if reuse is True: raise ValueError("PartitionedVariable %s does not exist, or was not " "created with tf.get_variable(). Did you mean to set " - "reuse=None in VarScope?" % name) + "reuse=False or reuse=tf.AUTO_REUSE in VarScope?" % name) slice_dim, slice_shape = _compute_slice_dim_and_shape( shape.as_list(), partitions) -- GitLab From a494558127ada7d32d2b85d99bebadc57c7f6e33 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Thu, 26 Oct 2017 21:13:19 -0700 Subject: [PATCH 445/573] Automated g4 rollback of changelist 172654120 PiperOrigin-RevId: 173630195 --- tensorflow/python/ops/array_ops.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 97dc63ebb1..ba8c611f57 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -2466,14 +2466,9 @@ def where(condition, x=None, y=None, name=None): """ if x is None and y is None: with ops.name_scope(name, "Where", [condition]) as name: - # Temporarily create an old style WhereOp nodedef + Operation without the - # attribute "T". - # TODO(b/67720963): Roll this back when the issue is resolved. - condition = gen_math_ops.cast(condition, dtypes.bool) - output = gen_array_ops.where(input=condition, name=name) - if context.in_graph_mode(): - output.op._node_def.attr.clear() - return output + condition = ops.convert_to_tensor( + condition, preferred_dtype=dtypes.bool, name="condition") + return gen_array_ops.where(input=condition, name=name) elif x is not None and y is not None: return gen_math_ops._select(condition=condition, t=x, e=y, name=name) else: -- GitLab From 9c8a520b07d5789dc3e43b0698573da822617c81 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Fri, 27 Oct 2017 00:15:55 -0700 Subject: [PATCH 446/573] Add WriteEvent method to SummaryWriterInterface Another change will follow that adds an op for this method. It will be useful for loading event logs into other types of summary writer implementations, like a database. This change might also make the new summary file writer go faster, due to less memory copying. PiperOrigin-RevId: 173640116 --- tensorflow/core/kernels/summary_interface.cc | 77 +++++++++++-------- tensorflow/core/kernels/summary_interface.h | 4 + .../core/kernels/summary_interface_test.cc | 26 +++++-- 3 files changed, 69 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc index a0b9038787..313137ae49 100644 --- a/tensorflow/core/kernels/summary_interface.cc +++ b/tensorflow/core/kernels/summary_interface.cc @@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/kernels/summary_interface.h" + +#include #include "tensorflow/compiler/xla/ptr_util.h" #include "tensorflow/core/framework/op_kernel.h" @@ -19,12 +22,10 @@ limitations under the License. #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/kernels/summary_interface.h" #include "tensorflow/core/lib/histogram/histogram.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/png/png_io.h" #include "tensorflow/core/lib/wav/wav_io.h" -#include "tensorflow/core/util/event.pb.h" #include "tensorflow/core/util/events_writer.h" namespace tensorflow { @@ -250,28 +251,34 @@ class SummaryWriterImpl : public SummaryWriterInterface { Status WriteTensor(int64 global_step, Tensor t, const string& tag, const string& serialized_metadata) override { - Summary s; - Summary::Value* v = s.add_value(); + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary::Value* v = e->mutable_summary()->add_value(); t.AsProtoTensorContent(v->mutable_tensor()); v->set_tag(tag); v->mutable_metadata()->ParseFromString(serialized_metadata); - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteScalar(int64 global_step, Tensor t, const string& tag) override { - Summary s; - Summary::Value* v = s.add_value(); + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary::Value* v = e->mutable_summary()->add_value(); v->set_tag(tag); float value; TF_RETURN_IF_ERROR(TensorValueAt(t, 0, &value)); v->set_simple_value(value); - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteHistogram(int64 global_step, Tensor t, const string& tag) override { - Summary s; - Summary::Value* v = s.add_value(); + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary::Value* v = e->mutable_summary()->add_value(); v->set_tag(tag); histogram::Histogram histo; for (int64 i = 0; i < t.NumElements(); i++) { @@ -287,7 +294,7 @@ class SummaryWriterImpl : public SummaryWriterInterface { } histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteImage(int64 global_step, Tensor tensor, const string& tag, @@ -306,7 +313,10 @@ class SummaryWriterImpl : public SummaryWriterInterface { return errors::InvalidArgument("Tensor too large for summary ", tensor.shape().DebugString()); } - Summary s; + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary* s = e->mutable_summary(); // The casts and h * w cannot overflow because of the limits above. const int batch_size = static_cast(tensor.dim_size(0)); const int h = static_cast(tensor.dim_size(1)); @@ -321,20 +331,20 @@ class SummaryWriterImpl : public SummaryWriterInterface { &values(i, 0, 0), Eigen::DSizes(hw, depth)); }; TF_RETURN_IF_ERROR( - AddImages(tag, max_images, batch_size, w, h, depth, ith_image, &s)); + AddImages(tag, max_images, batch_size, w, h, depth, ith_image, s)); } else if (tensor.dtype() == DT_HALF) { TF_RETURN_IF_ERROR(NormalizeAndAddImages( - tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s)); + tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s)); } else if (tensor.dtype() == DT_FLOAT) { TF_RETURN_IF_ERROR(NormalizeAndAddImages( - tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s)); + tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s)); } else { return errors::InvalidArgument( "Only DT_INT8, DT_HALF, and DT_FLOAT images are supported. Got ", DataTypeString(tensor.dtype())); } - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteAudio(int64 global_step, Tensor tensor, const string& tag, @@ -346,10 +356,13 @@ class SummaryWriterImpl : public SummaryWriterInterface { const int64 length_frames = tensor.dim_size(1); const int64 num_channels = tensor.dims() == 2 ? 1 : tensor.dim_size(tensor.dims() - 1); - Summary s; + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary* s = e->mutable_summary(); const int N = std::min(max_outputs, batch_size); for (int i = 0; i < N; ++i) { - Summary::Value* v = s.add_value(); + Summary::Value* v = s->add_value(); if (max_outputs > 1) { v->set_tag(strings::StrCat(tag, "/audio/", i)); } else { @@ -375,16 +388,12 @@ class SummaryWriterImpl : public SummaryWriterInterface { channels_by_frames.data(), sample_rate_truncated, num_channels, length_frames, sa->mutable_encoded_audio_string())); } - - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } - string DebugString() override { return "SummaryWriterImpl"; } - - private: - Status Enqueue(int64 global_step, const Summary& summary) { + Status WriteEvent(std::unique_ptr event) override { mutex_lock ml(mu_); - queue_.emplace_back(global_step, summary, env_->NowMicros()); + queue_.emplace_back(std::move(event)); if (queue_.size() >= max_queue_ || env_->NowMicros() - last_flush_ > 1000 * flush_millis_) { return InternalFlush(); @@ -392,13 +401,16 @@ class SummaryWriterImpl : public SummaryWriterInterface { return Status::OK(); } + string DebugString() override { return "SummaryWriterImpl"; } + + private: + double GetWallTime() { + return static_cast(env_->NowMicros()) / 1.0e6; + } + Status InternalFlush() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - for (const EventInfo& e : queue_) { - Event event; - event.set_step(std::get<0>(e)); - *event.mutable_summary() = std::get<1>(e); - event.set_wall_time(static_cast(std::get<2>(e)) / 1.0e6); - events_writer_->WriteEvent(event); + for (const std::unique_ptr& e : queue_) { + events_writer_->WriteEvent(*e); } queue_.clear(); if (!events_writer_->Flush()) { @@ -413,9 +425,8 @@ class SummaryWriterImpl : public SummaryWriterInterface { const int flush_millis_; uint64 last_flush_; Env* env_; - using EventInfo = std::tuple; mutex mu_; - std::vector queue_ GUARDED_BY(mu_); + std::vector> queue_ GUARDED_BY(mu_); // A pointer to allow deferred construction. std::unique_ptr events_writer_ GUARDED_BY(mu_); std::vector> registered_summaries_ diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h index 1b5d0b2748..ccf3459e56 100644 --- a/tensorflow/core/kernels/summary_interface.h +++ b/tensorflow/core/kernels/summary_interface.h @@ -15,8 +15,10 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ #define TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ +#include #include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/util/event.pb.h" namespace tensorflow { @@ -43,6 +45,8 @@ class SummaryWriterInterface : public ResourceBase { virtual Status WriteAudio(int64 global_step, Tensor t, const string& tag, int max_outputs_, float sample_rate) = 0; + + virtual Status WriteEvent(std::unique_ptr e) = 0; }; // Creates a SummaryWriterInterface instance which writes to a file. It will diff --git a/tensorflow/core/kernels/summary_interface_test.cc b/tensorflow/core/kernels/summary_interface_test.cc index 379e045ca3..58e021a0b3 100644 --- a/tensorflow/core/kernels/summary_interface_test.cc +++ b/tensorflow/core/kernels/summary_interface_test.cc @@ -12,11 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ - -#include +#include "tensorflow/core/kernels/summary_interface.h" #include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/kernels/summary_interface.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/refcount.h" #include "tensorflow/core/lib/io/path.h" @@ -43,8 +41,8 @@ class SummaryInterfaceTest : public ::testing::Test { protected: Status SummaryTestHelper( const string& test_name, - std::function writer_fn, - std::function test_fn) { + const std::function& writer_fn, + const std::function& test_fn) { static std::set* tests = new std::set(); CHECK(tests->insert(test_name).second) << ": " << test_name; @@ -182,6 +180,24 @@ TEST_F(SummaryInterfaceTest, WriteAudio) { })); } +TEST_F(SummaryInterfaceTest, WriteEvent) { + TF_CHECK_OK( + SummaryTestHelper("event_test", + [](SummaryWriterInterface* writer) { + std::unique_ptr e{new Event}; + e->set_step(7); + e->mutable_summary()->add_value()->set_tag("hi"); + TF_RETURN_IF_ERROR(writer->WriteEvent(std::move(e))); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 7); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "hi"); + })); +} + TEST_F(SummaryInterfaceTest, WallTime) { env_.AdvanceByMillis(7023); TF_CHECK_OK(SummaryTestHelper( -- GitLab From 37d483fda09a4e5f0580e5fe4a5d9b98cd7f02b8 Mon Sep 17 00:00:00 2001 From: Sergii Khomenko Date: Fri, 27 Oct 2017 11:10:44 +0200 Subject: [PATCH 447/573] Fix a typo --- tensorflow/contrib/gan/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md index 5d74df3ef7..3ab8478070 100644 --- a/tensorflow/contrib/gan/README.md +++ b/tensorflow/contrib/gan/README.md @@ -47,7 +47,7 @@ such as the Wasserstein loss, gradient penalty, mutual information penalty, etc * [evaluation](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/eval/python/): Use `Inception Score` or `Frechet Distance` with a pretrained Inception -network to evaluate your unconditional generative model. You can also also use +network to evaluate your unconditional generative model. You can also use your own pretrained classifier for more specific performance numbers, or use other methods for evaluating conditional generative models. -- GitLab From 3595d1613d0d46fad7cda0140965472351ff84b1 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 15 Oct 2017 23:07:30 +0000 Subject: [PATCH 448/573] Add `double` support for `tf.decode_csv` In the current tensorflow `tf.decode_csv` accepts `float`, `int32`, `int64`, `string` but not `double`. It seems adding `double` support makes sense as `StringToNumber` already support `double` type. This fix adds `double` support for `tf.decode_csv` Signed-off-by: Yong Tang --- tensorflow/core/kernels/decode_csv_op.cc | 19 +++++++++++++++++++ tensorflow/core/ops/parsing_ops.cc | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc index 5e48ae9766..6080f32072 100644 --- a/tensorflow/core/kernels/decode_csv_op.cc +++ b/tensorflow/core/kernels/decode_csv_op.cc @@ -137,6 +137,25 @@ class DecodeCSVOp : public OpKernel { } break; } + case DT_DOUBLE: { + // If this field is empty or NA value, check if default is given: + // If yes, use default value; Otherwise report error. + if (fields[f].empty() || fields[f] == na_value_) { + OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1, + errors::InvalidArgument( + "Field ", f, + " is required but missing in record ", i, "!")); + output[f]->flat()(i) = record_defaults[f].flat()(0); + } else { + double value; + OP_REQUIRES(ctx, strings::safe_strtod(fields[f].c_str(), &value), + errors::InvalidArgument( + "Field ", f, " in record ", i, + " is not a valid double: ", fields[f])); + output[f]->flat()(i) = value; + } + break; + } case DT_STRING: { // If this field is empty or NA value, check if default is given: // If yes, use default value; Otherwise report error. diff --git a/tensorflow/core/ops/parsing_ops.cc b/tensorflow/core/ops/parsing_ops.cc index b44ea2e080..40ec792ef8 100644 --- a/tensorflow/core/ops/parsing_ops.cc +++ b/tensorflow/core/ops/parsing_ops.cc @@ -329,7 +329,7 @@ REGISTER_OP("DecodeCSV") .Input("records: string") .Input("record_defaults: OUT_TYPE") .Output("output: OUT_TYPE") - .Attr("OUT_TYPE: list({float,int32,int64,string})") + .Attr("OUT_TYPE: list({float,double,int32,int64,string})") .Attr("field_delim: string = ','") .Attr("use_quote_delim: bool = true") .Attr("na_value: string = ''") -- GitLab From 73aaed655b4fddbd23c0dca32deb84e5dc191f0b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 15 Oct 2017 23:10:31 +0000 Subject: [PATCH 449/573] Update docs for `double` support on `tf.decode_csv` Signed-off-by: Yong Tang --- tensorflow/python/ops/parsing_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py index ea7132791c..14aef01dec 100644 --- a/tensorflow/python/ops/parsing_ops.py +++ b/tensorflow/python/ops/parsing_ops.py @@ -1183,7 +1183,7 @@ def decode_csv(records, record_defaults, field_delim=",", Each string is a record/row in the csv and all records should have the same format. record_defaults: A list of `Tensor` objects with specific types. - Acceptable types are `float32`, `int32`, `int64`, `string`. + Acceptable types are `float32`, `float64`, `int32`, `int64`, `string`. One tensor per column of the input record, with either a scalar default value for that column or empty if the column is required. field_delim: An optional `string`. Defaults to `","`. -- GitLab From 285ea39108cd7817c67abe5390b617c8cb6d8ccc Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 15 Oct 2017 23:11:04 +0000 Subject: [PATCH 450/573] Add test cases for `double` support of `tf.decode_csv` Signed-off-by: Yong Tang --- .../python/kernel_tests/decode_csv_op_test.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/decode_csv_op_test.py b/tensorflow/python/kernel_tests/decode_csv_op_test.py index 7d9e57c8e5..fec52fa9cc 100644 --- a/tensorflow/python/kernel_tests/decode_csv_op_test.py +++ b/tensorflow/python/kernel_tests/decode_csv_op_test.py @@ -34,7 +34,7 @@ class DecodeCSVOpTest(test.TestCase): out = sess.run(decode) for i, field in enumerate(out): - if field.dtype == np.float32: + if field.dtype == np.float32 or field.dtype == np.float64: self.assertAllClose(field, expected_out[i]) else: self.assertAllEqual(field, expected_out[i]) @@ -85,6 +85,17 @@ class DecodeCSVOpTest(test.TestCase): self._test(args, expected_out) + def testDouble(self): + args = { + "records": ["1.0", "-1.79e+308", '"1.79e+308"'], + "record_defaults": [np.array( + [], dtype=np.double)], + } + + expected_out = [[1.0, -1.79e+308, 1.79e+308]] + + self._test(args, expected_out) + def testInt64(self): args = { "records": ["1", "2", '"2147483648"'], -- GitLab From c6292a3f936daa4fdd92881ea1f6bec614c6bd06 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2017 13:31:56 +0000 Subject: [PATCH 451/573] Sanitize decode_csv_op.cc with `clang-format -i` Signed-off-by: Yong Tang --- tensorflow/core/kernels/decode_csv_op.cc | 43 ++++++++++++------------ 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc index 6080f32072..c4555db453 100644 --- a/tensorflow/core/kernels/decode_csv_op.cc +++ b/tensorflow/core/kernels/decode_csv_op.cc @@ -91,9 +91,9 @@ class DecodeCSVOp : public OpKernel { } else { int32 value; OP_REQUIRES(ctx, strings::safe_strto32(fields[f], &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid int32: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid int32: ", + fields[f])); output[f]->flat()(i) = value; } break; @@ -111,9 +111,9 @@ class DecodeCSVOp : public OpKernel { } else { int64 value; OP_REQUIRES(ctx, strings::safe_strto64(fields[f], &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid int64: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid int64: ", + fields[f])); output[f]->flat()(i) = value; } break; @@ -130,9 +130,9 @@ class DecodeCSVOp : public OpKernel { } else { float value; OP_REQUIRES(ctx, strings::safe_strtof(fields[f].c_str(), &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid float: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid float: ", + fields[f])); output[f]->flat()(i) = value; } break; @@ -145,13 +145,14 @@ class DecodeCSVOp : public OpKernel { errors::InvalidArgument( "Field ", f, " is required but missing in record ", i, "!")); - output[f]->flat()(i) = record_defaults[f].flat()(0); + output[f]->flat()(i) = + record_defaults[f].flat()(0); } else { double value; OP_REQUIRES(ctx, strings::safe_strtod(fields[f].c_str(), &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid double: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid double: ", + fields[f])); output[f]->flat()(i) = value; } break; @@ -207,10 +208,9 @@ class DecodeCSVOp : public OpKernel { if (!quoted) { while (static_cast(current_idx) < input.size() && input[current_idx] != delim_) { - OP_REQUIRES(ctx, - (!use_quote_delim_ || input[current_idx] != '"') && - input[current_idx] != '\n' && - input[current_idx] != '\r', + OP_REQUIRES(ctx, (!use_quote_delim_ || input[current_idx] != '"') && + input[current_idx] != '\n' && + input[current_idx] != '\r', errors::InvalidArgument( "Unquoted fields cannot have quotes/CRLFs inside")); field += input[current_idx]; @@ -238,11 +238,10 @@ class DecodeCSVOp : public OpKernel { } OP_REQUIRES( - ctx, - (static_cast(current_idx) < input.size() && - input[current_idx] == '"' && - (static_cast(current_idx) == input.size() - 1 || - input[current_idx + 1] == delim_)), + ctx, (static_cast(current_idx) < input.size() && + input[current_idx] == '"' && + (static_cast(current_idx) == input.size() - 1 || + input[current_idx + 1] == delim_)), errors::InvalidArgument("Quoted field has to end with quote " "followed by delim or end")); -- GitLab From 62a9ab28caef7bb6f3eb0c8b625c58fa8d88a173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E4=BC=A0=E6=AD=A6?= Date: Fri, 27 Oct 2017 08:52:21 -0500 Subject: [PATCH 452/573] fix broken link --- tensorflow/docs_src/programmers_guide/graphs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 10f53fe8f2..5ec3738d7d 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -100,7 +100,7 @@ to all API functions in the same context. For example: when run, will apply those gradients to a set of variables. Most programs rely solely on the default graph. However, -see [Dealing with multiple graphs](#dealing-with-multiple-graphs) for more +see [Dealing with multiple graphs](#programming_with_multiple_graphs) for more advanced use cases. High-level APIs such as the @{tf.estimator.Estimator} API manage the default graph on your behalf, and--for example--may create different graphs for training and evaluation. -- GitLab From 58d2c5f50508fad6591166f6e264d574f9c42768 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2017 07:31:33 -0700 Subject: [PATCH 453/573] Add `SANITY_STEPS_DESC` for do_clang_format_check (#14030) * Add `SANITY_STEPS_DESC` for do_clang_format_check This fix is a follow up to PR #13924 to add the corresponding description in `SANITY_STEPS_DESC`. See comment https://github.com/tensorflow/tensorflow/pull/13924#discussion_r147314599 for details. Signed-off-by: Yong Tang * Update description for Clang Format Check Signed-off-by: Yong Tang --- tensorflow/tools/ci_build/ci_sanity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 1e1fd7db6b..7e78def8eb 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -506,7 +506,7 @@ do_check_load_py_test() { # Supply all sanity step commands and descriptions SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_clang_format_check") -SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links") +SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Clang Format Check: Check .h and .cc files with Google C++ style") INCREMENTAL_FLAG="" DEFAULT_BAZEL_CONFIGS="--config=hdfs --config=gcp" -- GitLab From a7b8725271634e892781080464d0cef8516a9d36 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2017 14:47:27 +0000 Subject: [PATCH 454/573] Fix an ouput typo in `ci_sanity.sh` In the last PR #13924 (clang sanity check) the output message should be changed: `due to the absence of Python code changes` -> `due to the absence of .h or .cc code changes` Signed-off-by: Yong Tang --- tensorflow/tools/ci_build/ci_sanity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 1e1fd7db6b..9e23c6231a 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -460,7 +460,7 @@ do_clang_format_check() { if [[ -z "${CLANG_SRC_FILES}" ]]; then echo "do_clang_format_check will NOT run due to --incremental flag and "\ -"due to the absence of Python code changes in the last commit." +"due to the absence of .h or .cc code changes in the last commit." return 0 fi elif [[ -z "$1" ]]; then -- GitLab From 0ccf5cf600c0ab97a9e5d699caef5750d180348d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 08:15:17 -0700 Subject: [PATCH 455/573] Limit the amount of logspam a use of GraphKeys.VARIABLES causes. Multiple copies of this warning next to each other often make logs unreadable. PiperOrigin-RevId: 173672701 --- tensorflow/python/framework/ops.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index eceacb42d9..c278fb2a39 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -4938,9 +4938,10 @@ class GraphKeys(object): @decorator_utils.classproperty def VARIABLES(cls): # pylint: disable=no-self-argument - logging.warning("VARIABLES collection name is deprecated, " - "please use GLOBAL_VARIABLES instead; " - "VARIABLES will be removed after 2017-03-02.") + logging.log_first_n(logging.WARN, + "VARIABLES collection name is deprecated, please use " + "GLOBAL_VARIABLES instead; VARIABLES will be removed " + "after 2017-03-02.", 1) return cls.GLOBAL_VARIABLES -- GitLab From 4ae245a7db3d0457c4324ee7df8d020ba83b3c60 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 08:37:35 -0700 Subject: [PATCH 456/573] n/a (internal change only) PiperOrigin-RevId: 173674697 --- tensorflow/contrib/learn/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index f3949beed0..ac615b120c 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -768,7 +768,7 @@ py_test( ":learn", "//tensorflow/contrib/layers:layers_py", "//tensorflow/contrib/session_bundle:exporter", - "//tensorflow/contrib/session_bundle:manifest_proto_py", + "//tensorflow/contrib/session_bundle:manifest_proto_py_pb2", "//tensorflow/python:array_ops", "//tensorflow/python:client", "//tensorflow/python:client_testlib", -- GitLab From 4198e27be8115585ad6b5b141383fb7dc7856c24 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 09:00:51 -0700 Subject: [PATCH 457/573] [XLA:CPU] [XLA:GPU] Adds compiler support for C64 primitive type, including relevant elementwise unary and binary op lowering for CPU and GPU. We use a named LLVM struct "complex64", laid out the same as std::complex. This named struct is accessed via the llvm::Module, which required changes to accessors of PrimitiveTypeToIrType & friends. Ops that require atan2 (in particular, angle and log) are only supported on GPU at this point. LLVM lacks a CPU intrinsic for atan or atan2, whereas libdevice provides this for GPU. PiperOrigin-RevId: 173676849 --- .../jit/mark_for_compilation_pass_test.cc | 4 +- tensorflow/compiler/jit/xla_cpu_device.cc | 4 +- tensorflow/compiler/jit/xla_gpu_device.cc | 4 +- tensorflow/compiler/tests/BUILD | 7 +- tensorflow/compiler/tests/argminmax_test.py | 4 +- tensorflow/compiler/tests/binary_ops_test.py | 248 +++++++++--- tensorflow/compiler/tests/build_defs.bzl | 12 +- tensorflow/compiler/tests/gather_test.py | 12 +- tensorflow/compiler/tests/nary_ops_test.py | 20 + tensorflow/compiler/tests/random_ops_test.py | 8 +- tensorflow/compiler/tests/randomized_tests.cc | 345 +++++++++++------ tensorflow/compiler/tests/unary_ops_test.py | 130 ++++++- .../compiler/tests/variable_ops_test.py | 91 +++-- tensorflow/compiler/tests/xla_test.py | 11 +- .../tf2xla/kernels/batch_matmul_op.cc | 6 + .../compiler/tf2xla/kernels/binary_ops.cc | 14 +- tensorflow/compiler/tf2xla/kernels/cast_op.cc | 6 + .../compiler/tf2xla/kernels/gather_op.cc | 2 +- .../compiler/tf2xla/kernels/matmul_op.cc | 5 +- .../compiler/tf2xla/kernels/training_ops.cc | 23 +- .../compiler/tf2xla/kernels/unary_ops.cc | 9 + tensorflow/compiler/tf2xla/xla_helpers.cc | 6 + tensorflow/compiler/tf2xla/xla_op_registry.h | 15 +- .../xla/client/computation_builder.cc | 27 ++ .../compiler/xla/client/computation_builder.h | 19 + tensorflow/compiler/xla/literal_util.cc | 6 + .../xla/service/algebraic_simplifier.cc | 21 + .../xla/service/algebraic_simplifier_test.cc | 50 +++ .../xla/service/cpu/dot_op_emitter.cc | 47 ++- .../xla/service/cpu/elemental_ir_emitter.cc | 4 +- .../xla/service/cpu/ir_emission_utils.cc | 6 + .../compiler/xla/service/cpu/ir_emitter.cc | 64 +-- .../compiler/xla/service/dfs_hlo_visitor.h | 14 + .../xla/service/elemental_ir_emitter.cc | 363 ++++++++++++++++-- .../xla/service/elemental_ir_emitter.h | 13 + .../xla/service/gpu/elemental_ir_emitter.cc | 123 +++++- .../xla/service/gpu/elemental_ir_emitter.h | 3 + .../xla/service/gpu/hlo_to_ir_bindings.cc | 8 +- .../xla/service/gpu/hlo_to_ir_bindings.h | 5 +- .../compiler/xla/service/gpu/ir_emitter.cc | 79 ++-- .../compiler/xla/service/gpu/ir_emitter.h | 1 + .../xla/service/gpu/ir_emitter_nested.cc | 8 +- .../xla/service/gpu/ir_emitter_unnested.cc | 21 +- .../compiler/xla/service/hlo_evaluator.cc | 313 ++++++++++++--- .../compiler/xla/service/hlo_graph_dumper.cc | 4 + .../compiler/xla/service/hlo_instruction.cc | 67 +++- tensorflow/compiler/xla/service/hlo_opcode.cc | 8 + tensorflow/compiler/xla/service/hlo_opcode.h | 4 + .../compiler/xla/service/hlo_pass_pipeline.cc | 1 + .../compiler/xla/service/hlo_verifier.cc | 4 + .../xla/service/instruction_fusion.cc | 15 +- .../xla/service/llvm_ir/fused_ir_emitter.cc | 6 +- .../xla/service/llvm_ir/fused_ir_emitter.h | 4 +- .../compiler/xla/service/llvm_ir/ir_array.cc | 7 +- .../compiler/xla/service/llvm_ir/llvm_util.cc | 78 +++- .../compiler/xla/service/llvm_ir/llvm_util.h | 6 +- .../compiler/xla/service/llvm_ir/tuple_ops.cc | 14 +- .../compiler/xla/service/llvm_ir/tuple_ops.h | 8 +- .../compiler/xla/service/shape_inference.cc | 53 ++- .../xla/service/shape_inference_test.cc | 39 ++ .../compiler/xla/service/user_computation.cc | 8 + tensorflow/compiler/xla/shape_util.cc | 1 + .../xla/tests/client_library_test_base.h | 15 +- .../compiler/xla/tests/dot_operation_test.cc | 8 +- .../compiler/xla/tests/unary_op_test.cc | 80 +++- .../compiler/xla/tools/parser/hlo_parser.cc | 4 + tensorflow/compiler/xla/types.h | 4 +- tensorflow/compiler/xla/xla_data.proto | 14 +- 68 files changed, 2115 insertions(+), 518 deletions(-) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index 579ce415c5..b3d258aea1 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -144,8 +144,8 @@ TEST(XlaCompilationTest, UnsupportedTypes) { Node* a = ops::SourceOp( "Const", builder.opts() .WithName("A") - .WithAttr("dtype", DT_COMPLEX64) - .WithAttr("value", Tensor(DT_COMPLEX64, TensorShape()))); + .WithAttr("dtype", DT_COMPLEX128) + .WithAttr("value", Tensor(DT_COMPLEX128, TensorShape()))); Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B")); ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C")); TF_EXPECT_OK(builder.ToGraph(graph.get())); diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc index 2e33fdca65..e238252751 100644 --- a/tensorflow/compiler/jit/xla_cpu_device.cc +++ b/tensorflow/compiler/jit/xla_cpu_device.cc @@ -50,8 +50,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_CPU, XlaCpuDeviceFactory); // Kernel registrations -constexpr std::array kAllXlaCpuTypes = { - {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kAllXlaCpuTypes = { + {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}}; REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_CPU, XlaLocalLaunchOp, kAllXlaCpuTypes); REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_CPU, kAllXlaCpuTypes); diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc index 5233665ec2..2326070358 100644 --- a/tensorflow/compiler/jit/xla_gpu_device.cc +++ b/tensorflow/compiler/jit/xla_gpu_device.cc @@ -55,8 +55,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_GPU, XlaGpuDeviceFactory); // Kernel registrations -constexpr std::array kAllXlaGpuTypes = { - {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kAllXlaGpuTypes = { + {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}}; REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_GPU, XlaLocalLaunchOp, kAllXlaGpuTypes); REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_GPU, kAllXlaGpuTypes); diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 0eed475140..d07bf98296 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -23,6 +23,10 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test") load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test") load("//tensorflow/compiler/tests:build_defs.bzl", "generate_backend_suites") +load( + "//tensorflow/core:platform/default/build_config_root.bzl", + "tf_cuda_tests_tags", +) generate_backend_suites() @@ -581,11 +585,12 @@ cc_library( tf_cuda_cc_test( name = "randomized_tests", + size = "large", # This test is randomized, so only run it if explicitly requested. tags = [ "manual", "notap", - ], + ] + tf_cuda_tests_tags(), deps = [":randomized_tests_library"], ) diff --git a/tensorflow/compiler/tests/argminmax_test.py b/tensorflow/compiler/tests/argminmax_test.py index c2ce121348..ec547e16cd 100644 --- a/tensorflow/compiler/tests/argminmax_test.py +++ b/tensorflow/compiler/tests/argminmax_test.py @@ -46,7 +46,9 @@ class ArgMinMaxTest(xla_test.XLATestCase): self.assertAllEqual(result, expected) def testArgMinMax(self): - for dtype in self.numeric_types: + # Complex numbers do not support argmin/argmax. + minmax_types = set(self.numeric_types) - set(self.complex_types) + for dtype in minmax_types: self._assertOpOutputMatchesExpected( lambda x: math_ops.argmax(x, axis=0, output_type=dtypes.int32), np.array([1, 10, 27, 3, 3, 4], dtype=dtype), diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 9a225b32f8..d412c572ae 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -94,6 +94,15 @@ class BinaryOpsTest(XLATestCase): dtype(4), expected=np.array([[16], [81]], dtype=dtype)) + atan2_supported = self.device == "XLA_GPU" + if atan2_supported: + self._testBinary( + math_ops.atan2, + np.array([0, np.sqrt(2), 1, np.sqrt(2), 0], dtype), + np.array([1, np.sqrt(2), 0, -np.sqrt(2), -1], dtype), + expected=np.array( + [0, np.pi / 4, np.pi / 2, np.pi * 3 / 4, np.pi], dtype=dtype)) + self._testBinary( gen_math_ops._reciprocal_grad, np.array([4, -3, -2, 1], dtype=dtype), @@ -259,37 +268,38 @@ class BinaryOpsTest(XLATestCase): dtype(7), expected=np.array([[-6], [-5]], dtype=dtype)) - self._testBinary( - math_ops.maximum, - np.array([1, 2], dtype=dtype), - np.array([10, 20], dtype=dtype), - expected=np.array([10, 20], dtype=dtype)) - self._testBinary( - math_ops.maximum, - dtype(5), - np.array([1, 20], dtype=dtype), - expected=np.array([5, 20], dtype=dtype)) - self._testBinary( - math_ops.maximum, - np.array([[10], [2]], dtype=dtype), - dtype(7), - expected=np.array([[10], [7]], dtype=dtype)) + if dtype not in self.complex_types: # min/max not supported for complex + self._testBinary( + math_ops.maximum, + np.array([1, 2], dtype=dtype), + np.array([10, 20], dtype=dtype), + expected=np.array([10, 20], dtype=dtype)) + self._testBinary( + math_ops.maximum, + dtype(5), + np.array([1, 20], dtype=dtype), + expected=np.array([5, 20], dtype=dtype)) + self._testBinary( + math_ops.maximum, + np.array([[10], [2]], dtype=dtype), + dtype(7), + expected=np.array([[10], [7]], dtype=dtype)) - self._testBinary( - math_ops.minimum, - np.array([1, 20], dtype=dtype), - np.array([10, 2], dtype=dtype), - expected=np.array([1, 2], dtype=dtype)) - self._testBinary( - math_ops.minimum, - dtype(5), - np.array([1, 20], dtype=dtype), - expected=np.array([1, 5], dtype=dtype)) - self._testBinary( - math_ops.minimum, - np.array([[10], [2]], dtype=dtype), - dtype(7), - expected=np.array([[7], [2]], dtype=dtype)) + self._testBinary( + math_ops.minimum, + np.array([1, 20], dtype=dtype), + np.array([10, 2], dtype=dtype), + expected=np.array([1, 2], dtype=dtype)) + self._testBinary( + math_ops.minimum, + dtype(5), + np.array([1, 20], dtype=dtype), + expected=np.array([1, 5], dtype=dtype)) + self._testBinary( + math_ops.minimum, + np.array([[10], [2]], dtype=dtype), + dtype(7), + expected=np.array([[7], [2]], dtype=dtype)) self._testBinary( math_ops.multiply, @@ -307,21 +317,23 @@ class BinaryOpsTest(XLATestCase): dtype(7), expected=np.array([[70], [14]], dtype=dtype)) - self._testBinary( - math_ops.squared_difference, - np.array([1, 2], dtype=dtype), - np.array([10, 20], dtype=dtype), - expected=np.array([81, 324], dtype=dtype)) - self._testBinary( - math_ops.squared_difference, - dtype(5), - np.array([1, 2], dtype=dtype), - expected=np.array([16, 9], dtype=dtype)) - self._testBinary( - math_ops.squared_difference, - np.array([[1], [2]], dtype=dtype), - dtype(7), - expected=np.array([[36], [25]], dtype=dtype)) + # Complex support for squared_difference is incidental, see b/68205550 + if dtype not in self.complex_types: + self._testBinary( + math_ops.squared_difference, + np.array([1, 2], dtype=dtype), + np.array([10, 20], dtype=dtype), + expected=np.array([81, 324], dtype=dtype)) + self._testBinary( + math_ops.squared_difference, + dtype(5), + np.array([1, 2], dtype=dtype), + expected=np.array([16, 9], dtype=dtype)) + self._testBinary( + math_ops.squared_difference, + np.array([[1], [2]], dtype=dtype), + dtype(7), + expected=np.array([[36], [25]], dtype=dtype)) self._testBinary( nn_ops.bias_add, @@ -334,6 +346,139 @@ class BinaryOpsTest(XLATestCase): np.array([2, -1], dtype=dtype), expected=np.array([[[[3, 1], [5, 3]]]], dtype=dtype)) + def testComplexOps(self): + for dtype in self.complex_types: + ctypes = {np.complex64: np.float32} + self._testBinary( + math_ops.complex, + np.array([[[[-1, 2], [2, 0]]]], dtype=ctypes[dtype]), + np.array([[[[2, -3], [0, 4]]]], dtype=ctypes[dtype]), + expected=np.array([[[[-1 + 2j, 2 - 3j], [2, 4j]]]], dtype=dtype)) + + self._testBinary( + lambda x, y: math_ops.approximate_equal(x, y, tolerance=0.0001), + np.array( + [[[[-1 + 2j, 2.00009999 - 3j], [2 - 3j, 3 + 4.01j]]]], + dtype=dtype), + np.array( + [[[[-1.001 + 2j, 2 - 3j], [2 - 3.00009j, 3 + 4j]]]], dtype=dtype), + expected=np.array([[[[False, True], [True, False]]]], dtype=dtype)) + + self._testBinary( + gen_math_ops._real_div, + np.array([3, 3j, -1.5j, -8, 2 + 3j, 2 + 4j, 44 + 3j], dtype=dtype), + np.array([2, -2, 7j, -4j, 4 - 6j, 1 + 2j, 0], dtype=dtype), + expected=np.array( + [ + 1.5, -1.5j, -0.2142857, -2j, (2 + 3j) / (4 - 6j), 2, + float("inf") + ], + dtype=dtype)) + + # TODO(b/65408531): support+test pow for cplx + + lhs = np.array([4 + 2j, -3 - 1j, 2j, 1], dtype=dtype) + rhs = np.array([5, -6j, 7 - 3j, -8j], dtype=dtype) + self._testBinary( + gen_math_ops._reciprocal_grad, lhs, rhs, expected=-rhs * lhs * lhs) + + self._testBinary( + gen_math_ops._sigmoid_grad, lhs, rhs, expected=rhs * lhs * (1 - lhs)) + + # TODO(b/65408531): support+test _rsqrt_grad for cplx (needs pow) + + self._testBinary( + gen_math_ops._sqrt_grad, lhs, rhs, expected=rhs / (2 * lhs)) + + self._testBinary( + gen_math_ops._tanh_grad, lhs, rhs, expected=rhs * (1 - lhs * lhs)) + + def testComplexMath(self): + for dtype in self.complex_types: + self._testBinary( + math_ops.add, + np.array([1 + 3j, 2 + 7j], dtype=dtype), + np.array([10 - 4j, 20 + 17j], dtype=dtype), + expected=np.array([11 - 1j, 22 + 24j], dtype=dtype)) + self._testBinary( + math_ops.add, + dtype(5 - 7j), + np.array([1 + 2j, 2 + 4j], dtype=dtype), + expected=np.array([6 - 5j, 7 - 3j], dtype=dtype)) + self._testBinary( + math_ops.add, + np.array([[1 - 2j], [2 + 1j]], dtype=dtype), + dtype(7 + 5j), + expected=np.array([[8 + 3j], [9 + 6j]], dtype=dtype)) + + self._testBinary( + math_ops.subtract, + np.array([1 + 3j, 2 + 7j], dtype=dtype), + np.array([10 - 4j, 20 + 17j], dtype=dtype), + expected=np.array([-9 + 7j, -18 - 10j], dtype=dtype)) + self._testBinary( + math_ops.subtract, + dtype(5 - 7j), + np.array([1 + 2j, 2 + 4j], dtype=dtype), + expected=np.array([4 - 9j, 3 - 11j], dtype=dtype)) + self._testBinary( + math_ops.subtract, + np.array([[1 - 2j], [2 + 1j]], dtype=dtype), + dtype(7 + 5j), + expected=np.array([[-6 - 7j], [-5 - 4j]], dtype=dtype)) + + self._testBinary( + math_ops.multiply, + np.array([1 + 3j, 2 + 7j], dtype=dtype), + np.array([10 - 4j, 20 + 17j], dtype=dtype), + expected=np.array( + [(1 + 3j) * (10 - 4j), (2 + 7j) * (20 + 17j)], dtype=dtype)) + self._testBinary( + math_ops.multiply, + dtype(5 - 7j), + np.array([1 + 2j, 2 + 4j], dtype=dtype), + expected=np.array( + [(5 - 7j) * (1 + 2j), (5 - 7j) * (2 + 4j)], dtype=dtype)) + self._testBinary( + math_ops.multiply, + np.array([[1 - 2j], [2 + 1j]], dtype=dtype), + dtype(7 + 5j), + expected=np.array( + [[(7 + 5j) * (1 - 2j)], [(7 + 5j) * (2 + 1j)]], dtype=dtype)) + + self._testBinary( + math_ops.div, + np.array([8 - 1j, 2 + 16j], dtype=dtype), + np.array([2 + 4j, 4 - 8j], dtype=dtype), + expected=np.array( + [(8 - 1j) / (2 + 4j), (2 + 16j) / (4 - 8j)], dtype=dtype)) + self._testBinary( + math_ops.div, + dtype(1 + 2j), + np.array([2 + 4j, 4 - 8j], dtype=dtype), + expected=np.array( + [(1 + 2j) / (2 + 4j), (1 + 2j) / (4 - 8j)], dtype=dtype)) + self._testBinary( + math_ops.div, + np.array([2 + 4j, 4 - 8j], dtype=dtype), + dtype(1 + 2j), + expected=np.array( + [(2 + 4j) / (1 + 2j), (4 - 8j) / (1 + 2j)], dtype=dtype)) + + # TODO(b/68205550): math_ops.squared_difference shouldn't be supported. + + self._testBinary( + nn_ops.bias_add, + np.array([[1 + 2j, 2 + 7j], [3 - 5j, 4 + 2j]], dtype=dtype), + np.array([2 + 6j, -1 - 3j], dtype=dtype), + expected=np.array([[3 + 8j, 1 + 4j], [5 + 1j, 3 - 1j]], dtype=dtype)) + self._testBinary( + nn_ops.bias_add, + np.array([[[[1 + 4j, 2 - 1j], [3 + 7j, 4]]]], dtype=dtype), + np.array([2 + 1j, -1 + 2j], dtype=dtype), + expected=np.array( + [[[[3 + 5j, 1 + 1j], [5 + 8j, 3 + 2j]]]], dtype=dtype)) + def _testDivision(self, dtype): """Test cases for division operators.""" self._testBinary( @@ -352,18 +497,19 @@ class BinaryOpsTest(XLATestCase): dtype(2), expected=np.array([[5], [2]], dtype=dtype)) - self._testBinary( - gen_math_ops._floor_div, - np.array([3, 3, -1, -9, -8], dtype=dtype), - np.array([2, -2, 7, 2, -4], dtype=dtype), - expected=np.array([1, -2, -1, -5, 2], dtype=dtype)) + if dtype not in self.complex_types: # floordiv unsupported for complex. + self._testBinary( + gen_math_ops._floor_div, + np.array([3, 3, -1, -9, -8], dtype=dtype), + np.array([2, -2, 7, 2, -4], dtype=dtype), + expected=np.array([1, -2, -1, -5, 2], dtype=dtype)) def testIntDivision(self): for dtype in self.int_types: self._testDivision(dtype) def testFloatDivision(self): - for dtype in self.float_types: + for dtype in self.float_types + self.complex_types: self._testDivision(dtype) def _testRemainder(self, dtype): diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl index a56c53de0f..0528a5415d 100644 --- a/tensorflow/compiler/tests/build_defs.bzl +++ b/tensorflow/compiler/tests/build_defs.bzl @@ -49,11 +49,15 @@ def tf_xla_py_test(name, srcs=[], deps=[], tags=[], data=[], main=None, backend_deps = [] backend_data = [] if backend == "cpu": - backend_args += ["--test_device=XLA_CPU", - "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL"] + backend_args += [ + "--test_device=XLA_CPU", + "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64" + ] elif backend == "gpu": - backend_args += ["--test_device=XLA_GPU", - "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL"] + backend_args += [ + "--test_device=XLA_GPU", + "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64" + ] backend_tags += ["requires-gpu-sm35"] elif backend in plugins: backend_args += ["--test_device=" + plugins[backend]["device"], diff --git a/tensorflow/compiler/tests/gather_test.py b/tensorflow/compiler/tests/gather_test.py index 4b81c1d7ab..664c77f200 100644 --- a/tensorflow/compiler/tests/gather_test.py +++ b/tensorflow/compiler/tests/gather_test.py @@ -30,8 +30,6 @@ from tensorflow.python.platform import test FLAGS = flags.FLAGS -_TEST_TYPES = [dtypes.float32] - class GatherTest(xla_test.XLATestCase): @@ -46,7 +44,7 @@ class GatherTest(xla_test.XLATestCase): def testScalar1D(self): with self.test_session() as session, self.test_scope(): data = np.array([0, 1, 2, 3, 7, 5]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for indices in 4, [1, 2, 2, 4, 5]: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -60,7 +58,7 @@ class GatherTest(xla_test.XLATestCase): with self.test_session() as session, self.test_scope(): data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], [12, 13, 14]]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, -1: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -74,7 +72,7 @@ class GatherTest(xla_test.XLATestCase): with self.test_session() as session, self.test_scope(): data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], [12, 13, 14]]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, -1: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -94,7 +92,7 @@ class GatherTest(xla_test.XLATestCase): [12, 13, 14]]) # The indices must be in bounds for any axis. indices_np = np.array([0, 1, 0, 2]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, -1: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -112,7 +110,7 @@ class GatherTest(xla_test.XLATestCase): """Check that scalar and empty indices shapes work as well.""" shape = (2, 1, 3, 2) for indices_shape in (), (0,), (2, 0), (2, 3): - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, 2, 3, -1, -2: params = self._buildParams(np.random.randn(*shape), dtype) indices = np.random.randint(shape[axis], size=indices_shape) diff --git a/tensorflow/compiler/tests/nary_ops_test.py b/tensorflow/compiler/tests/nary_ops_test.py index ae60d78f1a..e4843b169b 100644 --- a/tensorflow/compiler/tests/nary_ops_test.py +++ b/tensorflow/compiler/tests/nary_ops_test.py @@ -68,6 +68,26 @@ class NAryOpsTest(XLATestCase): np.array([42], dtype=np.float32)], expected=np.array([48], dtype=np.float32)) + def testComplex(self): + for dtype in self.complex_types: + self._testNAry( + math_ops.add_n, [np.array([[1 + 2j, 2 - 3j, 3 + 4j]], dtype=dtype)], + expected=np.array([[1 + 2j, 2 - 3j, 3 + 4j]], dtype=dtype)) + + self._testNAry( + math_ops.add_n, [ + np.array([1 + 2j, 2 - 3j], dtype=dtype), + np.array([10j, 20], dtype=dtype) + ], + expected=np.array([1 + 12j, 22 - 3j], dtype=dtype)) + self._testNAry( + math_ops.add_n, [ + np.array([-4, 5j], dtype=dtype), + np.array([2 + 10j, -2], dtype=dtype), + np.array([42j, 3 + 3j], dtype=dtype) + ], + expected=np.array([-2 + 52j, 1 + 8j], dtype=dtype)) + @unittest.skip("IdentityN is temporarily CompilationOnly as workaround") def testIdentityN(self): self._testNAryLists(array_ops.identity_n, diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py index a17a3f3d65..d6c93088d4 100644 --- a/tensorflow/compiler/tests/random_ops_test.py +++ b/tensorflow/compiler/tests/random_ops_test.py @@ -29,6 +29,9 @@ from tensorflow.python.platform import googletest class RandomOpsTest(XLATestCase): """Test cases for random-number generating operators.""" + def _random_types(self): + return set(self.numeric_types) - set(self.complex_types) + def _testRngIsNotConstant(self, rng, dtype): # Tests that 'rng' does not always return the same value. with self.test_session() as sess: @@ -51,7 +54,8 @@ class RandomOpsTest(XLATestCase): def rng(dtype): return random_ops.random_uniform(shape=[2], dtype=dtype, maxval=1000000) - for dtype in self.numeric_types: + + for dtype in self._random_types(): self._testRngIsNotConstant(rng, dtype) def testRandomNormalIsNotConstant(self): @@ -63,7 +67,7 @@ class RandomOpsTest(XLATestCase): self._testRngIsNotConstant(rng, dtype) def testRandomUniformIsInRange(self): - for dtype in self.numeric_types: + for dtype in self._random_types(): with self.test_session() as sess: with self.test_scope(): x = random_ops.random_uniform(shape=[1000], dtype=dtype, minval=-2, diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc index 5129171cd4..461af83362 100644 --- a/tensorflow/compiler/tests/randomized_tests.cc +++ b/tensorflow/compiler/tests/randomized_tests.cc @@ -75,7 +75,7 @@ namespace { // Command line flags: see main() below. int64 tf_xla_random_seed = 0; int32 tf_xla_test_repetitions = 20; -int64 tf_xla_max_tensor_size = 100000LL; +int64 tf_xla_max_tensor_size = 10000LL; string* tf_xla_test_device_ptr; // initial value set in main() bool tf_xla_test_use_jit = true; @@ -83,8 +83,8 @@ string LocalDeviceToFullDeviceName(const string& device) { return strings::StrCat("/job:localhost/replica:0/task:0/device:", device); } -constexpr std::array kAllXlaTypes = { - {DT_INT32, DT_FLOAT, DT_BOOL}}; +constexpr std::array kAllXlaTypes = { + {DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}}; // An OpTestBuilder is a graph builder class that takes as input an operator to // test, its inputs and attributes, and builds a graph that executes the @@ -449,6 +449,13 @@ Tensor OpTest::RandomTensor(DataType dtype, gtl::ArraySlice shape) { }); break; } + case DT_COMPLEX64: { + std::uniform_real_distribution distribution(-1.0f, 1.0f); + test::FillFn(&tensor, [this, &distribution](int i) { + return complex64(distribution(generator()), distribution(generator())); + }); + break; + } case DT_INT32: { std::uniform_int_distribution distribution(-(1 << 20), 1 << 20); test::FillFn(&tensor, [this, &distribution](int i) -> int32 { @@ -624,11 +631,47 @@ std::vector OpTest::AsInt32s(const std::vector& int64s) { // Functions for comparing tensors. +template +double Abs(T x) { + return std::fabs(x); +} + +template <> +double Abs(complex64 x) { + return std::abs(x); +} + template bool IsClose(const T& x, const T& y, double atol, double rtol) { if (std::isnan(x) && std::isnan(y)) return true; if (x == y) return true; // Allow inf == inf. - return fabs(x - y) < atol + rtol * fabs(x); + return Abs(x - y) < atol + rtol * Abs(x); +} + +template <> +bool IsClose(const complex64& x, const complex64& y, double atol, + double rtol) { + if (std::isnan(x.real()) && std::isnan(y.real())) { + if (std::isnan(x.imag()) && std::isnan(y.imag())) { + return true; + } + if (x.imag() == y.imag()) return true; // Allow inf == inf. + return Abs(x.imag() - y.imag()) < atol + rtol * Abs(x.imag()); + } else if (std::isnan(x.imag()) && std::isnan(y.imag())) { + if (x.real() == y.real()) return true; // Allow inf == inf. + return Abs(x.real() - y.real()) < atol + rtol * Abs(x.real()); + } + if (x == y) return true; // Allow inf == inf. + return Abs(x - y) < atol + rtol * Abs(x); +} + +template +string Str(T x) { + return strings::StrCat(x); +} +template <> +string Str(complex64 x) { + return strings::StrCat("(", x.real(), ", ", x.imag(), ")"); } template @@ -639,9 +682,10 @@ Status TensorsAreCloseImpl(const Tensor& x, const Tensor& y, double atol, for (int i = 0; i < Tx.size(); ++i) { if (!IsClose(Tx(i), Ty(i), atol, rtol)) { return errors::InvalidArgument(strings::StrCat( - i, "-th tensor element isn't close: ", Tx(i), " vs. ", Ty(i), - ". x = ", x.DebugString(), "y = ", y.DebugString(), "atol = ", atol, - " rtol = ", rtol, " tol = ", atol + rtol * std::fabs(Tx(i)))); + i, "-th tensor element isn't close: ", Str(Tx(i)), " vs. ", + Str(Ty(i)), ". x = ", x.DebugString(), "y = ", y.DebugString(), + "atol = ", atol, " rtol = ", rtol, + " tol = ", atol + rtol * Abs(Tx(i)))); } } return Status::OK(); @@ -683,6 +727,8 @@ Status TensorsAreClose(const Tensor& a, const Tensor& b, double atol, return TensorsAreCloseImpl(a, b, atol, rtol); case DT_DOUBLE: return TensorsAreCloseImpl(a, b, atol, rtol); + case DT_COMPLEX64: + return TensorsAreCloseImpl(a, b, atol, rtol); case DT_INT32: return TensorsAreEqualImpl(a, b); case DT_INT64: @@ -822,7 +868,7 @@ Tensor AsIntTensor(DataType dtype, const std::vector& values) { TEST_F(OpTest, Abs) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Abs").RandomInput(type).Attr("T", type)); }); @@ -837,7 +883,7 @@ TEST_F(OpTest, Acosh) { TEST_F(OpTest, Add) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Add") .RandomInput(type, dims.first) @@ -848,7 +894,7 @@ TEST_F(OpTest, Add) { TEST_F(OpTest, AddN) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); int n = std::uniform_int_distribution(1, 5)(generator()); auto shape = RandomDims(); @@ -890,9 +936,10 @@ TEST_F(OpTest, Any) { TEST_F(OpTest, ApproximateEqual) { Repeatedly([this]() { auto dims = RandomDims(); + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ApproximateEqual") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) + .RandomInput(type, dims) + .RandomInput(type, dims) .Attr("T", DT_FLOAT)); }); } @@ -1038,6 +1085,7 @@ TEST_F(OpTest, AvgPool3DGrad) { TEST_F(OpTest, BatchMatMul) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); std::vector output_dims = RandomDims(2, 5, 0, 7); int64 ndims = output_dims.size(); int64 inner_dim = RandomDim(); @@ -1056,9 +1104,9 @@ TEST_F(OpTest, BatchMatMul) { } return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchMatMul") - .RandomInput(DT_FLOAT, x_dims) - .RandomInput(DT_FLOAT, y_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, x_dims) + .RandomInput(type, y_dims) + .Attr("T", type) .Attr("adj_x", adj_x) .Attr("adj_y", adj_y)); }); @@ -1090,10 +1138,11 @@ TEST_F(OpTest, BatchToSpace) { CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals), TensorShape({num_block_dims, 2}))); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchToSpace") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(crops) - .Attr("T", DT_FLOAT) + .Attr("T", type) .Attr("block_size", block_size)); }); } @@ -1127,13 +1176,14 @@ TEST_F(OpTest, BatchToSpaceND) { CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals), TensorShape({num_block_dims, 2}))); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("BatchToSpaceND") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(test::AsTensor( std::vector(block_dims.begin(), block_dims.end()))) .Input(crops) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } @@ -1142,18 +1192,20 @@ TEST_F(OpTest, BiasAdd) { auto x_dims = RandomDims(2, kDefaultMaxRank); auto y_dims = {x_dims[x_dims.size() - 1]}; // TODO(phawkins): test both data formats. + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAdd") - .RandomInput(DT_FLOAT, x_dims) - .RandomInput(DT_FLOAT, y_dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, x_dims) + .RandomInput(type, y_dims) + .Attr("T", type)); }); } TEST_F(OpTest, BiasAddGrad) { Repeatedly([this]() { // TODO(phawkins): test both data formats. + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("BiasAddGrad").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("BiasAddGrad").RandomInput(type).Attr("T", type)); }); } @@ -1161,10 +1213,11 @@ TEST_F(OpTest, BiasAddV1) { Repeatedly([this]() { auto x_dims = RandomDims(2, kDefaultMaxRank); auto y_dims = {x_dims[x_dims.size() - 1]}; + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAddV1") - .RandomInput(DT_FLOAT, x_dims) - .RandomInput(DT_FLOAT, y_dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, x_dims) + .RandomInput(type, y_dims) + .Attr("T", type)); }); } @@ -1221,8 +1274,8 @@ TEST_F(OpTest, BroadcastGradientArgs) { TEST_F(OpTest, Cast) { Repeatedly([this]() { DataType src_type, dst_type; - src_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL}); - dst_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL}); + src_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}); + dst_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Cast") .RandomInput(src_type) .Attr("SrcT", src_type) @@ -1293,11 +1346,12 @@ TEST_F(OpTest, Conv2D) { std::vector kernel_dims = {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out}; + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv2D") - .RandomInput(DT_FLOAT, data_dims) - .RandomInput(DT_FLOAT, kernel_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, data_dims) + .RandomInput(type, kernel_dims) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); @@ -1317,12 +1371,13 @@ TEST_F(OpTest, Conv2DBackpropFilter) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); Tensor kernel_shape = test::AsTensor(AsInt32s( {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out})); + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv2DBackpropFilter") - .RandomInput(DT_FLOAT, activations) + .RandomInput(type, activations) .Input(kernel_shape) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); @@ -1342,12 +1397,13 @@ TEST_F(OpTest, Conv2DBackpropInput) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out}; + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv2DBackpropInput") .Input(in_shape) - .RandomInput(DT_FLOAT, kernel) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, kernel) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); @@ -1365,11 +1421,12 @@ TEST_F(OpTest, Conv3D) { std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out}; + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3D") - .RandomInput(DT_FLOAT, data) - .RandomInput(DT_FLOAT, kernel) - .Attr("T", DT_FLOAT) + .RandomInput(type, data) + .RandomInput(type, kernel) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); @@ -1389,12 +1446,13 @@ TEST_F(OpTest, Conv3DBackpropFilter) { Tensor kernel_shape = test::AsTensor( AsInt32s({d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out})); + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3DBackpropFilterV2") - .RandomInput(DT_FLOAT, activations) + .RandomInput(type, activations) .Input(kernel_shape) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); @@ -1413,17 +1471,34 @@ TEST_F(OpTest, Conv3DBackpropInput) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out}; + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3DBackpropInputV2") .Input(in_shape) - .RandomInput(DT_FLOAT, kernel) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, kernel) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); } +TEST_F(OpTest, Cos) { + Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("Cos").RandomInput(type).Attr("T", type)); + }); +} + +TEST_F(OpTest, Cosh) { + Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("Cosh").RandomInput(type).Attr("T", type)); + }); +} + TEST_F(OpTest, DepthToSpace) { Repeatedly([this]() { int64 block = RandomDim(2, 5); @@ -1431,14 +1506,16 @@ TEST_F(OpTest, DepthToSpace) { input_dims[1] = (input_dims[1] + (block - 1)) / block; input_dims[2] = (input_dims[2] + (block - 1)) / block; input_dims[3] *= block * block; + DataType type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("DepthToSpace") - .RandomInput(DT_FLOAT, input_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, input_dims) + .Attr("T", type) .Attr("block_size", block)); }); } TEST_F(OpTest, DepthwiseConv2DNative) { + if (1) return; Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); std::uniform_int_distribution random_int(1, 5); @@ -1449,17 +1526,20 @@ TEST_F(OpTest, DepthwiseConv2DNative) { std::vector kernel_dims = {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier}; + std::vector strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims); + strides[2] = strides[1]; // Current impl only supports equal strides return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("DepthwiseConv2dNative") .RandomInput(DT_FLOAT, input_dims) .RandomInput(DT_FLOAT, kernel_dims) .Attr("T", DT_FLOAT) - .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) + .Attr("strides", strides) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); } TEST_F(OpTest, DepthwiseConv2DBackpropFilter) { + if (1) return; Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); std::uniform_int_distribution random_int(1, 5); @@ -1472,33 +1552,22 @@ TEST_F(OpTest, DepthwiseConv2DBackpropFilter) { FORMAT_NHWC, batch, features_in * depth_multiplier, d.output_dims); Tensor kernel_shape = test::AsTensor(AsInt32s( {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier})); + std::vector strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims); + strides[2] = strides[1]; // Current impl only supports equal strides return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("DepthwiseConv2dNativeBackpropFilter") .RandomInput(DT_FLOAT, activations) .Input(kernel_shape) .RandomInput(DT_FLOAT, backprop) .Attr("T", DT_FLOAT) - .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) + .Attr("strides", strides) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); }); } -TEST_F(OpTest, Cos) { - Repeatedly([this]() { - return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Cos").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); - }); -} - -TEST_F(OpTest, Cosh) { - Repeatedly([this]() { - return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Cosh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); - }); -} - TEST_F(OpTest, DepthwiseConv2DBackpropInput) { + if (1) return; Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); std::uniform_int_distribution random_int(1, 5); @@ -1511,21 +1580,24 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) { FORMAT_NHWC, batch, features_in * depth_multiplier, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier}; + std::vector strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims); + strides[2] = strides[1]; // Current impl only supports equal strides return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("DepthwiseConv2dNativeBackpropInput") .Input(in_shape) .RandomInput(DT_FLOAT, kernel) .RandomInput(DT_FLOAT, backprop) .Attr("T", DT_FLOAT) - .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) + .Attr("strides", strides) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); }); } TEST_F(OpTest, Diag) { + if (1) return; Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose(kAllXlaTypes); std::vector dims; // Diag causes a quadratic blowup in output size. int64 size; @@ -1540,7 +1612,7 @@ TEST_F(OpTest, Diag) { TEST_F(OpTest, DiagPart) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose(kAllXlaTypes); auto dims = RandomDims(1, 3); // Duplicate the random dims. std::vector doubled_dims(dims.size() * 2); @@ -1554,7 +1626,7 @@ TEST_F(OpTest, DiagPart) { TEST_F(OpTest, Div) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Div") .RandomInput(type, dims.first) @@ -1650,7 +1722,7 @@ TEST_F(OpTest, SeluGrad) { TEST_F(OpTest, Equal) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Equal") .RandomInput(type, dims.first) @@ -1661,15 +1733,17 @@ TEST_F(OpTest, Equal) { TEST_F(OpTest, Exp) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Exp").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Exp").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Expm1) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Expm1").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Expm1").RandomInput(type).Attr("T", type)); }); } @@ -1809,15 +1883,17 @@ TEST_F(OpTest, LinSpace) { TEST_F(OpTest, Log) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Log").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Log").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Log1p) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Log1p").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Log1p").RandomInput(type).Attr("T", DT_FLOAT)); }); } @@ -1914,10 +1990,11 @@ TEST_F(OpTest, MatMul) { std::swap(b_dims[0], b_dims[1]); } + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatMul") - .RandomInput(DT_FLOAT, a_dims) - .RandomInput(DT_FLOAT, b_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, a_dims) + .RandomInput(type, b_dims) + .Attr("T", type) .Attr("transpose_a", transpose_a) .Attr("transpose_b", transpose_b)); }); @@ -1925,7 +2002,7 @@ TEST_F(OpTest, MatMul) { TEST_F(OpTest, MatrixDiag) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiag") .RandomInput(type, RandomDims(1)) .Attr("T", type)); @@ -1934,7 +2011,7 @@ TEST_F(OpTest, MatrixDiag) { TEST_F(OpTest, MatrixDiagPart) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiagPart") .RandomInput(type, RandomDims(2)) .Attr("T", type)); @@ -2025,7 +2102,7 @@ TEST_F(OpTest, MaxPool3D) { TEST_F(OpTest, Mean) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); // TODO(phawkins): CPU and XLA differ output for reducing across a // size-0 dimension (nan vs 0). For now, require size >= 1. std::vector data_dims = RandomDims(0, kDefaultMaxRank, 1); @@ -2076,7 +2153,7 @@ TEST_F(OpTest, Mod) { TEST_F(OpTest, Mul) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Mul") .RandomInput(type, dims.first) @@ -2087,7 +2164,7 @@ TEST_F(OpTest, Mul) { TEST_F(OpTest, Neg) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Neg").RandomInput(type).Attr("T", type)); }); @@ -2095,7 +2172,7 @@ TEST_F(OpTest, Neg) { TEST_F(OpTest, NotEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("NotEqual") .RandomInput(type, dims.first) @@ -2136,7 +2213,7 @@ TEST_F(OpTest, OneHot) { TEST_F(OpTest, OnesLike) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("OnesLike").RandomInput(type).Attr("T", type)); }); @@ -2195,16 +2272,17 @@ TEST_F(OpTest, Pow) { // nontermination. Repeatedly([this]() { auto dims = BroadcastableDims(); + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Pow") - .RandomInput(DT_FLOAT, dims.first) - .RandomInput(DT_FLOAT, dims.second) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims.first) + .RandomInput(type, dims.second) + .Attr("T", type)); }); } TEST_F(OpTest, Prod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2238,7 +2316,7 @@ TEST_F(OpTest, Range) { TEST_F(OpTest, Rank) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Rank").RandomInput(type).Attr("T", type)); }); @@ -2246,7 +2324,7 @@ TEST_F(OpTest, Rank) { TEST_F(OpTest, RealDiv) { Repeatedly([this]() { - DataType type = DT_FLOAT; + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RealDiv") .RandomInput(type, dims.first) @@ -2257,18 +2335,20 @@ TEST_F(OpTest, RealDiv) { TEST_F(OpTest, Reciprocal) { Repeatedly([this]() { + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Reciprocal").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Reciprocal").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, ReciprocalGrad) { Repeatedly([this]() { std::vector dims = RandomDims(); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReciprocalGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } TEST_F(OpTest, Relu) { @@ -2335,24 +2415,24 @@ TEST_F(OpTest, Reshape) { TEST_F(OpTest, Reverse) { Repeatedly([this]() { std::vector dims = RandomDims(1); - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose(kAllXlaTypes); int64 rank = dims.size(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Reverse") .RandomInput(type, dims) .RandomInput(DT_BOOL, {rank}) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } TEST_F(OpTest, ReverseV2) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReverseV2") .RandomInput(type, data_dims) .Input(indices) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } @@ -2372,18 +2452,20 @@ TEST_F(OpTest, Round) { TEST_F(OpTest, Rsqrt) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Rsqrt").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Rsqrt").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, RsqrtGrad) { Repeatedly([this]() { auto dims = RandomDims(); + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RsqrtGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } @@ -2411,24 +2493,26 @@ TEST_F(OpTest, ShapeN) { TEST_F(OpTest, Sigmoid) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sigmoid").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sigmoid").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, SigmoidGrad) { Repeatedly([this]() { auto dims = RandomDims(); + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SigmoidGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } TEST_F(OpTest, Sign) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sign").RandomInput(type).Attr("T", type)); }); @@ -2436,21 +2520,23 @@ TEST_F(OpTest, Sign) { TEST_F(OpTest, Sin) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sin").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sin").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Sinh) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sinh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sinh").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Size) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Size").RandomInput(type).Attr("T", type)); }); @@ -2562,10 +2648,11 @@ TEST_F(OpTest, SpaceToBatch) { CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals), TensorShape({num_block_dims, 2}))); + DataType type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SpaceToBatch") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(paddings) - .Attr("T", DT_FLOAT) + .Attr("T", type) .Attr("block_size", block_size)); }); } @@ -2603,13 +2690,14 @@ TEST_F(OpTest, SpaceToBatchND) { CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals), TensorShape({num_block_dims, 2}))); + DataType type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("SpaceToBatchND") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(test::AsTensor( std::vector(block_dims.begin(), block_dims.end()))) .Input(paddings) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } @@ -2699,18 +2787,20 @@ TEST_F(OpTest, Split) { TEST_F(OpTest, Sqrt) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sqrt").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sqrt").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, SqrtGrad) { Repeatedly([this]() { auto dims = RandomDims(); + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SqrtGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } @@ -2726,7 +2816,7 @@ TEST_F(OpTest, SquaredDifference) { TEST_F(OpTest, Square) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Square").RandomInput(type).Attr("T", type)); }); @@ -2752,7 +2842,7 @@ TEST_F(OpTest, Squeeze) { TEST_F(OpTest, Sub) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Sub") .RandomInput(type, dims.first) @@ -2763,7 +2853,7 @@ TEST_F(OpTest, Sub) { TEST_F(OpTest, Sum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2875,25 +2965,28 @@ TEST_F(OpTest, StridedSliceGrad) { TEST_F(OpTest, Tan) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Tan").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Tan").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Tanh) { Repeatedly([this]() { + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Tanh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Tanh").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, TanhGrad) { Repeatedly([this]() { auto dims = RandomDims(); + DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TanhGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } @@ -2951,7 +3044,7 @@ TEST_F(OpTest, TruncateMod) { TEST_F(OpTest, ZerosLike) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("ZerosLike").RandomInput(type).Attr("T", type)); }); diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index 71221b284d..76644380bd 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -328,6 +328,131 @@ class UnaryOpsTest(XLATestCase): np.array([-1, -0.5, 0, 0.3], dtype=dtype), expected=np.array([-1, -64.0 / 127, 0, 38.0 / 127], dtype=dtype)) + def testComplexOps(self): + for dtype in self.complex_types: + # TODO(b/65408531): math_ops.acosh (needs pow) + # TODO(b/65408531): math_ops.asinh (needs pow) + + # TODO(b/65408531): Wider support for log (needs atan2). + atan2_supported = self.device == "XLA_GPU" + if atan2_supported: + self._assertOpOutputMatchesExpected( + math_ops.atanh, + np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype), + expected=np.arctanh( + np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.cosh, + np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype), + expected=np.cosh(np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.sinh, + np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype), + expected=np.sinh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.exp, + np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype), + expected=np.exp(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.expm1, + np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype), + expected=np.expm1(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.reciprocal, + np.array([[1, 2j, 2 + 3j]], dtype=dtype), + expected=1.0 / np.array([[1, 2j, 2 + 3j]], dtype=dtype)) + + if atan2_supported: + self._assertOpOutputMatchesExpected( + math_ops.log, + np.array([[5j, 3 - 2j]], dtype=dtype), + expected=np.log(np.array([[5j, 3 - 2j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.sin, + np.array([[5j, 3 - 2j]], dtype=dtype), + expected=np.sin(np.array([[5j, 3 - 2j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.cos, + np.array([[5j, 3 - 2j]], dtype=dtype), + expected=np.cos(np.array([[5j, 3 - 2j]], dtype=dtype))) + + # TODO(b/34703906): improve log1p implementation and make tolerance + # tighter. + if atan2_supported: # TODO(b/34703906): log support + self._assertOpOutputMatchesExpected( + math_ops.log1p, + np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype), + expected=np.log1p( + np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype))) + + # TODO(b/34703906): math_ops.rsqrt (needs pow) + + # TODO(b/34703906): math_ops.sigmoid (needs tanh) + + # TODO(b/34703906): math_ops.sqrt (needs pow) + + self._assertOpOutputMatchesExpected( + math_ops.tan, + np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype), + expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype))) + + # TODO(b/34703906): math_ops.tanh (as itself) + + ctypes = {np.complex64: np.float32} + self._assertOpOutputMatchesExpected( + math_ops.abs, + np.array([[3 - 4j, -1j, np.inf]], dtype=dtype), + expected=np.array([[5, 1, np.inf]], dtype=ctypes[dtype])) + + self._assertOpOutputMatchesExpected( + math_ops.negative, + np.array([[-1 + 2j, -3j]], dtype=dtype), + expected=np.array([[1 - 2j, 3j]], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + math_ops.square, + np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype), + expected=np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype)**2) + + self._assertOpOutputMatchesExpected( + array_ops.zeros_like, + np.array([[4j, 3 - 2j], [2, -1j]], dtype=dtype), + expected=np.array([[0, 0], [0, 0]], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + array_ops.ones_like, + np.array([[-4j, 3 + 2j], [2, -1j]], dtype=dtype), + expected=np.array([[1, 1], [1, 1]], dtype=dtype)) + + if atan2_supported: # TODO(b/34703906): atan2 support + self._assertOpOutputMatchesExpected( + math_ops.angle, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.angle( + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.conj, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.array([1 - 3j, -4 - 7j, 2.7, 3j], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + math_ops.imag, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.array([3, 7, 0, -3], dtype=ctypes[dtype])) + + self._assertOpOutputMatchesExpected( + math_ops.real, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.array([1, -4, 2.7, 0], dtype=ctypes[dtype])) + def testIntOps(self): for dtype in self.int_types: self._assertOpOutputMatchesExpected( @@ -399,11 +524,14 @@ class UnaryOpsTest(XLATestCase): def testCast(self): shapes = [[], [4], [2, 3], [2, 0, 4]] - types = [dtypes.bool, dtypes.int32, dtypes.float32] + types = [dtypes.bool, dtypes.int32, dtypes.float32] + self.complex_tf_types for shape in shapes: for src_type in types: for dst_type in types: src = np.arange(np.prod(shape)).astype(src_type.as_numpy_dtype) + if src_type in self.complex_tf_types: + src += (np.arange(np.prod(shape)) * 2j).astype( + src_type.as_numpy_dtype) src = src.reshape(shape) dst = src.astype(dst_type.as_numpy_dtype) diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py index fdf3f9fb6a..c50342dee4 100644 --- a/tensorflow/compiler/tests/variable_ops_test.py +++ b/tensorflow/compiler/tests/variable_ops_test.py @@ -43,7 +43,7 @@ class VariableOpsTest(XLATestCase): # Regression test for a bug where computations with one non-constant # output and one variable update were mishandled. for dtype in self.numeric_types: - init = np.array([[1, 2], [3, 4]], dtype=dtype) + init = np.array([[1, 2j], [3, 4]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) @@ -51,82 +51,91 @@ class VariableOpsTest(XLATestCase): x = v.assign_add(p) with ops.control_dependencies([x]): y = v.read_value() - self.assertAllClose(np.array([[2, 3], [4, 5]], dtype=dtype), - sess.run(y, {p: 1})) + self.assertAllClose( + np.array([[2, 1 + 2j], [4, 5]]).astype(dtype), sess.run(y, { + p: 1 + })) def testSparseRead0DIndices(self): for dtype in self.numeric_types: - init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) + init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8j, 9, 10, + 11]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read(2) - self.assertAllClose(np.array([8, 9, 10, 11], dtype=dtype), sess.run(x)) + self.assertAllClose( + np.array([8j, 9, 10, 11]).astype(dtype), sess.run(x)) def testSparseRead1DIndices(self): for dtype in self.numeric_types: - init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) + init = np.array([[0, 1, 2, 3], [4, 5, 6j, 7], [8, 9, 10, + 11]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([2, 1]) self.assertAllClose( - np.array([[8, 9, 10, 11], [4, 5, 6, 7]], dtype=dtype), sess.run(x)) + np.array([[8, 9, 10, 11], [4, 5, 6j, 7]]).astype(dtype), + sess.run(x)) def testSparseRead2DIndices(self): for dtype in self.numeric_types: - init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) + init = np.array([[0, 1, 2j, 3], [4, 5, 6, 7], [8, 9, 10, + 11]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([[2, 1], [0, 2]]) self.assertAllClose( - np.array( - [[[8, 9, 10, 11], [4, 5, 6, 7]], [[0, 1, 2, 3], [8, 9, 10, - 11]]], - dtype=dtype), sess.run(x)) + np.array([[[8, 9, 10, 11], [4, 5, 6, 7]], + [[0, 1, 2j, 3], [8, 9, 10, 11]]]).astype(dtype), + sess.run(x)) def testSparseRead2DIndices3DTensor(self): for dtype in self.numeric_types: - init = np.array( - [[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]], - [[20, 21, 22], [23, 24, 25]], [[30, 31, 32], [33, 34, 35]]], - dtype=dtype) + init = np.array([[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]], + [[20, 21, 22], [23, 24j, 25]], + [[30, 31, 32], [33, 34, 35]]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([[2, 1], [3, 0]]) self.assertAllClose( np.array( - [[[[20, 21, 22], [23, 24, 25]], [[10, 11, 12], [13, 14, 15]]], + [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]], [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]]], - dtype=dtype), sess.run(x)) + ).astype(dtype), sess.run(x)) def testReadWrite(self): """Tests initialization, reading, and writing a resource variable.""" - with self.test_session() as session: - with self.test_scope(): - with variable_scope.variable_scope("ascope", use_resource=True): - x = variable_scope.get_variable( - "x", - shape=[], - dtype=dtypes.float32, - initializer=init_ops.constant_initializer(2)) - a = x.read_value() - with ops.control_dependencies([a]): - b = state_ops.assign(x, 47) - with ops.control_dependencies([b]): - c = x.read_value() - with ops.control_dependencies([c]): - d = state_ops.assign_add(x, 3) - with ops.control_dependencies([d]): - e = x.read_value() - - session.run(variables.global_variables_initializer()) - v1, v2, v3 = session.run([a, c, e]) - self.assertAllClose(2.0, v1) - self.assertAllClose(47.0, v2) - self.assertAllClose(50.0, v3) + for dtype in self.numeric_types: + with self.test_session() as session: + print(ops.get_default_graph()) + with self.test_scope(): + with variable_scope.variable_scope("ascope", use_resource=True): + x = variable_scope.get_variable( + "x", + shape=[], + dtype=dtype, + initializer=init_ops.constant_initializer(2)) + a = x.read_value() + with ops.control_dependencies([a]): + b = state_ops.assign(x, dtype(47)) + with ops.control_dependencies([b]): + c = x.read_value() + with ops.control_dependencies([c]): + d = state_ops.assign_add(x, np.array(6 + 2j).astype(dtype)) + with ops.control_dependencies([d]): + e = state_ops.assign_sub(x, dtype(3)) + with ops.control_dependencies([e]): + f = x.read_value() + + session.run(variables.global_variables_initializer()) + v1, v2, v3 = session.run([a, c, f]) + self.assertAllClose(dtype(2), v1) + self.assertAllClose(dtype(47), v2) + self.assertAllClose(np.array(50 + 2j).astype(dtype), v3) def testTraining(self): """Tests a gradient descent step for a simple model.""" diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py index da6dc88f1f..0be127997e 100644 --- a/tensorflow/compiler/tests/xla_test.py +++ b/tensorflow/compiler/tests/xla_test.py @@ -63,12 +63,19 @@ class XLATestCase(test.TestCase): self.float_tf_types = [ dtype for dtype in self.all_tf_types if dtype.is_floating ] - self.numeric_tf_types = self.int_tf_types + self.float_tf_types + self.complex_tf_types = [ + dtype for dtype in self.all_tf_types if dtype.is_complex + ] + self.numeric_tf_types = ( + self.int_tf_types + self.float_tf_types + self.complex_tf_types) self.all_types = [dtype.as_numpy_dtype for dtype in self.all_tf_types] self.int_types = [dtype.as_numpy_dtype for dtype in self.int_tf_types] self.float_types = [dtype.as_numpy_dtype for dtype in self.float_tf_types] - self.numeric_types = self.int_types + self.float_types + self.complex_types = [ + dtype.as_numpy_dtype for dtype in self.complex_tf_types + ] + self.numeric_types = self.int_types + self.float_types + self.complex_types # Parse the manifest file, if any, into a regex identifying tests to # disable diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc index 16b778bca4..73ccc151c1 100644 --- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc @@ -77,7 +77,13 @@ class BatchMatMulOp : public XlaOpKernel { xla::ComputationBuilder* builder = ctx->builder(); xla::ComputationDataHandle x_handle = ctx->Input(0); + if (BaseType(input_type(0)) == DT_COMPLEX64 && adj_x_) { + x_handle = builder->Conj(x_handle); + } xla::ComputationDataHandle y_handle = ctx->Input(1); + if (BaseType(input_type(1)) == DT_COMPLEX64 && adj_y_) { + y_handle = builder->Conj(y_handle); + } // Reshape input tensors into 3D tensors by flattening the batch // dimensions. This makes it easier to unroll the batch dimension. diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc index 4673bbda14..1de9192432 100644 --- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Native XLA implementations of simple unary Ops +// Native XLA implementations of simple binary Ops #include "tensorflow/compiler/tf2xla/kernels/cwise_ops.h" #include "tensorflow/compiler/tf2xla/xla_helpers.h" @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/client_library.h" #include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" namespace tensorflow { namespace { @@ -50,6 +51,9 @@ XLA_MAKE_BINARY(Sub, b->Sub(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Mul, b->Mul(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Div, b->Div(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(Atan2, b->Atan2(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(Complex, b->Complex(lhs, rhs, extend_dimensions)); + // Implementation of FloorDiv. Pseudo-code: // if ((x < 0) != (y < 0)) { // T abs_x = std::abs(x); @@ -171,8 +175,12 @@ class ApproximateEqualOp : public XlaOpKernel { // Computes the max of the scalar input x and 0. void Compile(XlaOpKernelContext* ctx) override { xla::ComputationBuilder* b = ctx->builder(); - auto result = b->Lt(b->Abs(b->Sub(ctx->Input(0), ctx->Input(1))), - XlaHelpers::FloatLiteral(b, input_type(0), tolerance_)); + auto abs = b->Abs(b->Sub(ctx->Input(0), ctx->Input(1))); + auto abs_shape = b->GetShape(abs); + OP_REQUIRES_OK(ctx, abs_shape.status()); + auto abs_type = abs_shape.ValueOrDie()->element_type(); + auto result = b->Lt( + abs, b->ConvertElementType(b->ConstantR0(tolerance_), abs_type)); ctx->SetOutput(0, result); } diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc index 2331520230..43a6a747c6 100644 --- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/primitive_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" namespace tensorflow { @@ -40,6 +41,11 @@ class CastOp : public XlaOpKernel { output = input; } else if (dst_dtype_ == DT_BOOL) { output = builder->Ne(input, XlaHelpers::Zero(builder, src_dtype_)); + } else if (xla::primitive_util::IsComplexType(src_type_) && + !xla::primitive_util::IsComplexType(dst_type_)) { + // As in cast_op.h, we replicate the numpy behavior of truncating the + // imaginary part. + output = builder->ConvertElementType(builder->Real(input), dst_type_); } else { output = builder->ConvertElementType(input, dst_type_); } diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op.cc b/tensorflow/compiler/tf2xla/kernels/gather_op.cc index db449ec345..e420f21ca3 100644 --- a/tensorflow/compiler/tf2xla/kernels/gather_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/gather_op.cc @@ -192,7 +192,7 @@ void GatherOpDynamicSlice::Compile(XlaOpKernelContext* context) { errors::InvalidArgument("indices must be int32 or int64")); xla::ComputationDataHandle gather = XlaComputeGatherDynamicSlice( - context, input, input_shape, indices, indices_shape, axis, DT_FLOAT, + context, input, input_shape, indices, indices_shape, axis, input_type(0), index_type, builder); context->SetOutput(0, gather); } diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc index 5c799a0e4f..fcef497e58 100644 --- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc @@ -23,6 +23,9 @@ limitations under the License. namespace tensorflow { namespace { +constexpr std::array kMatmulTypes = { + {DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}}; + class MatMulOp : public XlaOpKernel { public: explicit MatMulOp(OpKernelConstruction* ctx, bool is_sparse = false) @@ -73,7 +76,7 @@ class MatMulOp : public XlaOpKernel { bool transpose_b_; }; -REGISTER_XLA_OP(Name("MatMul").TypeConstraint("T", kFloatTypes), MatMulOp); +REGISTER_XLA_OP(Name("MatMul").TypeConstraint("T", kMatmulTypes), MatMulOp); class SparseMatMulOp : public MatMulOp { public: diff --git a/tensorflow/compiler/tf2xla/kernels/training_ops.cc b/tensorflow/compiler/tf2xla/kernels/training_ops.cc index 82ae0df5cc..5534d1bfa1 100644 --- a/tensorflow/compiler/tf2xla/kernels/training_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/training_ops.cc @@ -37,8 +37,9 @@ class ResourceApplyGradientDescent : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->AssignVariable(0, ctx->input_type(1), handle)); } }; -REGISTER_XLA_OP(Name("ResourceApplyGradientDescent"), - ResourceApplyGradientDescent); +REGISTER_XLA_OP( + Name("ResourceApplyGradientDescent").TypeConstraint("T", kFloatTypes), + ResourceApplyGradientDescent); class ResourceApplyMomentum : public XlaOpKernel { public: @@ -109,7 +110,8 @@ class ResourceApplyMomentum : public XlaOpKernel { private: bool use_nesterov_; }; -REGISTER_XLA_OP(Name("ResourceApplyMomentum"), ResourceApplyMomentum); +REGISTER_XLA_OP(Name("ResourceApplyMomentum").TypeConstraint("T", kFloatTypes), + ResourceApplyMomentum); class ResourceApplyAdagrad : public XlaOpKernel { public: @@ -163,7 +165,8 @@ class ResourceApplyAdagrad : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->AssignVariable(1, type, accum)); } }; -REGISTER_XLA_OP(Name("ResourceApplyAdagrad"), ResourceApplyAdagrad); +REGISTER_XLA_OP(Name("ResourceApplyAdagrad").TypeConstraint("T", kFloatTypes), + ResourceApplyAdagrad); class ResourceApplyAdam : public XlaOpKernel { public: @@ -263,7 +266,8 @@ class ResourceApplyAdam : public XlaOpKernel { private: DataType dtype_; }; -REGISTER_XLA_OP(Name("ResourceApplyAdam"), ResourceApplyAdam); +REGISTER_XLA_OP(Name("ResourceApplyAdam").TypeConstraint("T", kFloatTypes), + ResourceApplyAdam); class ResourceApplyRMSProp : public XlaOpKernel { public: @@ -362,7 +366,8 @@ class ResourceApplyRMSProp : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->AssignVariable(2, type, new_mom)); } }; -REGISTER_XLA_OP(Name("ResourceApplyRMSProp"), ResourceApplyRMSProp); +REGISTER_XLA_OP(Name("ResourceApplyRMSProp").TypeConstraint("T", kFloatTypes), + ResourceApplyRMSProp); void CompileFtrl(XlaOpKernelContext* ctx, DataType dtype, bool has_l2_shrinkage) { @@ -500,7 +505,8 @@ class ResourceApplyFtrl : public XlaOpKernel { private: DataType dtype_; }; -REGISTER_XLA_OP(Name("ResourceApplyFtrl"), ResourceApplyFtrl); +REGISTER_XLA_OP(Name("ResourceApplyFtrl").TypeConstraint("T", kFloatTypes), + ResourceApplyFtrl); class ResourceApplyFtrlV2 : public XlaOpKernel { public: @@ -515,7 +521,8 @@ class ResourceApplyFtrlV2 : public XlaOpKernel { private: DataType dtype_; }; -REGISTER_XLA_OP(Name("ResourceApplyFtrlV2"), ResourceApplyFtrlV2); +REGISTER_XLA_OP(Name("ResourceApplyFtrlV2").TypeConstraint("T", kFloatTypes), + ResourceApplyFtrlV2); } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc index 651bbe2b40..b35f6fc2e0 100644 --- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc @@ -41,6 +41,12 @@ namespace { }; \ REGISTER_XLA_OP(Name(#NAME), NAME##Op); +XLAJIT_MAKE_UNARY(ComplexAbs, b->Abs(x)); + +XLAJIT_MAKE_UNARY(Angle, b->Atan2(b->Imag(x), b->Real(x))); + +XLAJIT_MAKE_UNARY(Conj, b->Complex(b->Real(x), b->Neg(b->Imag(x)))); + // Return x if x>0, otherwise -x. XLAJIT_MAKE_UNARY(Abs, b->Abs(x)); @@ -162,6 +168,9 @@ XLAJIT_MAKE_UNARY(Square, b->Mul(x, x)); XLAJIT_MAKE_UNARY(Tan, b->Div(b->Sin(x), b->Cos(x))); XLAJIT_MAKE_UNARY(Tanh, b->Tanh(x)); +XLAJIT_MAKE_UNARY(Real, b->Real(x)); +XLAJIT_MAKE_UNARY(Imag, b->Imag(x)); + #undef XLAJIT_MAKE_UNARY } // namespace diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc index f59b83cfdd..de5ad5f176 100644 --- a/tensorflow/compiler/tf2xla/xla_helpers.cc +++ b/tensorflow/compiler/tf2xla/xla_helpers.cc @@ -97,6 +97,9 @@ xla::ComputationDataHandle XlaHelpers::IntegerLiteral( case xla::F64: literal = *xla::Literal::CreateR0(value); break; + case xla::C64: + literal = *xla::Literal::CreateR0(value); + break; case xla::PRED: LOG(FATAL) << "pred element type is not integral"; case xla::S16: @@ -132,6 +135,9 @@ xla::ComputationDataHandle XlaHelpers::FloatLiteral(xla::ComputationBuilder* b, case xla::F64: return b->ConstantR0(value); break; + case xla::C64: + return b->ConstantR0(value); + break; default: LOG(FATAL) << "unhandled element type " << type; } diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h index 2144868646..6aee8c91cc 100644 --- a/tensorflow/compiler/tf2xla/xla_op_registry.h +++ b/tensorflow/compiler/tf2xla/xla_op_registry.h @@ -47,14 +47,17 @@ extern const char* const DEVICE_XLA_GPU; constexpr std::array kFloatTypes = { {DT_HALF, DT_FLOAT, DT_DOUBLE}}; -constexpr std::array kNumericTypes = { - {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE}}; +constexpr std::array kNumericTypes = { + {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, + DT_COMPLEX64}}; -constexpr std::array kCpuAllTypes = { - {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kCpuAllTypes = { + {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, + DT_COMPLEX64, DT_BOOL}}; -constexpr std::array kGpuAllTypes = { - {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kGpuAllTypes = { + {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, + DT_COMPLEX64, DT_BOOL}}; // Class that manages registrations of operators and devices for the XLA JIT. // Not thread-safe. diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc index b9977fb2f8..edf5a1822c 100644 --- a/tensorflow/compiler/xla/client/computation_builder.cc +++ b/tensorflow/compiler/xla/client/computation_builder.cc @@ -913,6 +913,17 @@ ComputationDataHandle ComputationBuilder::CustomCall( return ParseOpResponse(s, &response); } +ComputationDataHandle ComputationBuilder::Complex( + const ComputationDataHandle& real, const ComputationDataHandle& imag, + tensorflow::gtl::ArraySlice broadcast_dimensions) { + return BinaryOp(BINOP_COMPLEX, real, imag, broadcast_dimensions); +} + +ComputationDataHandle ComputationBuilder::Conj( + const ComputationDataHandle& operand) { + return Complex(Real(operand), Neg(Imag(operand))); +} + ComputationDataHandle ComputationBuilder::Add( const ComputationDataHandle& lhs, const ComputationDataHandle& rhs, tensorflow::gtl::ArraySlice broadcast_dimensions) { @@ -995,6 +1006,12 @@ ComputationDataHandle ComputationBuilder::Abs( return UnaryOp(UNOP_ABS, operand); } +ComputationDataHandle ComputationBuilder::Atan2( + const ComputationDataHandle& y, const ComputationDataHandle& x, + tensorflow::gtl::ArraySlice broadcast_dimensions) { + return BinaryOp(BINOP_ATAN2, y, x, broadcast_dimensions); +} + ComputationDataHandle ComputationBuilder::Exp( const ComputationDataHandle& operand) { return UnaryOp(UNOP_EXP, operand); @@ -1040,6 +1057,16 @@ ComputationDataHandle ComputationBuilder::Tanh( return UnaryOp(UNOP_TANH, operand); } +ComputationDataHandle ComputationBuilder::Real( + const ComputationDataHandle& operand) { + return UnaryOp(UNOP_REAL, operand); +} + +ComputationDataHandle ComputationBuilder::Imag( + const ComputationDataHandle& operand) { + return UnaryOp(UNOP_IMAG, operand); +} + ComputationDataHandle ComputationBuilder::IsFinite( const ComputationDataHandle& operand) { return UnaryOp(UNOP_IS_FINITE, operand); diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h index 93c2a80678..d2f0c7cff0 100644 --- a/tensorflow/compiler/xla/client/computation_builder.h +++ b/tensorflow/compiler/xla/client/computation_builder.h @@ -431,6 +431,14 @@ class ComputationBuilder { // of the operands is a scalar, or an explicit broadcast dimension is given // (see g3doc for more details). + // Enqueues a complex compose instruction onto the computation. + ComputationDataHandle Complex( + const ComputationDataHandle& real, const ComputationDataHandle& imag, + tensorflow::gtl::ArraySlice broadcast_dimensions = {}); + + // Enqueues a complex conjugate instruction onto the computation. + ComputationDataHandle Conj(const ComputationDataHandle& operand); + // Enqueues an add instruction onto the computation. ComputationDataHandle Add( const ComputationDataHandle& lhs, const ComputationDataHandle& rhs, @@ -542,6 +550,11 @@ class ComputationBuilder { // Enqueues an abs instruction onto the computation. ComputationDataHandle Abs(const ComputationDataHandle& operand); + // Enqueues a atan2 instruction onto the computation. + ComputationDataHandle Atan2( + const ComputationDataHandle& y, const ComputationDataHandle& x, + tensorflow::gtl::ArraySlice broadcast_dimensions = {}); + // Enqueues an exp instruction onto the computation. ComputationDataHandle Exp(const ComputationDataHandle& operand); @@ -570,6 +583,12 @@ class ComputationBuilder { // Enqueues a tanh instruction onto the computation. ComputationDataHandle Tanh(const ComputationDataHandle& operand); + // Enqueues a real-part instruction onto the computation. + ComputationDataHandle Real(const ComputationDataHandle& operand); + + // Enqueues an imaginary-part instruction onto the computation. + ComputationDataHandle Imag(const ComputationDataHandle& operand); + // Enqueues a float32 sqrt instruction onto the computation. // (float32 is specified as there is an implicit float32 0.5f constant // exponent). diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc index 413b85e3ba..8fc8644a60 100644 --- a/tensorflow/compiler/xla/literal_util.cc +++ b/tensorflow/compiler/xla/literal_util.cc @@ -204,6 +204,8 @@ Status Literal::Copy(const Literal& src_literal, return *Literal::CreateR0(0); case F64: return *Literal::CreateR0(0); + case C64: + return *Literal::CreateR0(0); case PRED: return *Literal::CreateR0(false); case S16: @@ -236,6 +238,8 @@ Status Literal::Copy(const Literal& src_literal, return *Literal::CreateR0(1); case F64: return *Literal::CreateR0(1); + case C64: + return *Literal::CreateR0(1); case PRED: return *Literal::CreateR0(true); case S16: @@ -271,6 +275,8 @@ Status Literal::Copy(const Literal& src_literal, case F64: return *Literal::CreateR0( -std::numeric_limits::infinity()); + case C64: + LOG(FATAL) << "C64 element type has no minimum value"; case PRED: return *Literal::CreateR0(false); case S16: diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 35ab4d89cc..2a610e91f0 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -141,6 +141,9 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { Status HandleConvert(HloInstruction* convert) override; + Status HandleReal(HloInstruction* real, HloInstruction* operand) override; + Status HandleImag(HloInstruction* imag, HloInstruction* operand) override; + Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, HloInstruction* rhs, const Window& window) override; @@ -967,6 +970,24 @@ Status AlgebraicSimplifierVisitor::HandleConvert(HloInstruction* convert) { return Status::OK(); } +// Real(Complex(r, i)) -> r +Status AlgebraicSimplifierVisitor::HandleReal(HloInstruction* real, + HloInstruction* operand) { + if (operand->opcode() == HloOpcode::kComplex) { + return ReplaceInstruction(real, operand->mutable_operand(0)); + } + return Status::OK(); +} + +// Imag(Complex(r, i)) -> i +Status AlgebraicSimplifierVisitor::HandleImag(HloInstruction* imag, + HloInstruction* operand) { + if (operand->opcode() == HloOpcode::kComplex) { + return ReplaceInstruction(imag, operand->mutable_operand(1)); + } + return Status::OK(); +} + Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) { // Eliminate nop pads (padding all zero), and replace a pad with negative // padding with a pad with non-negative padding followed by a slice. diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index 3df50080d1..87d4fc9663 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -433,6 +433,56 @@ TEST_F(AlgebraicSimplifierTest, DivOneArray) { EXPECT_EQ(root, param0); } +// Test that real(complex(r,i)) is simplified to r. +TEST_F(AlgebraicSimplifierTest, RealOfComplex) { + Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r2f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r2f32, "param1")); + HloInstruction* cplx = builder.AddInstruction( + HloInstruction::CreateBinary(ShapeUtil::ChangeElementType(r2f32, C64), + HloOpcode::kComplex, param0, param1)); + HloInstruction* real = builder.AddInstruction( + HloInstruction::CreateUnary(r2f32, HloOpcode::kReal, cplx)); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + HloInstruction* root = computation->root_instruction(); + EXPECT_EQ(root, real); + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + root = computation->root_instruction(); + EXPECT_EQ(root, param0); +} + +// Test that imag(complex(r,i)) is simplified to i. +TEST_F(AlgebraicSimplifierTest, ImagOfComplex) { + Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r2f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r2f32, "param1")); + HloInstruction* cplx = builder.AddInstruction( + HloInstruction::CreateBinary(ShapeUtil::ChangeElementType(r2f32, C64), + HloOpcode::kComplex, param0, param1)); + HloInstruction* imag = builder.AddInstruction( + HloInstruction::CreateUnary(r2f32, HloOpcode::kImag, cplx)); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + HloInstruction* root = computation->root_instruction(); + EXPECT_EQ(root, imag); + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + root = computation->root_instruction(); + EXPECT_EQ(root, param1); +} + // Test that get_element(make_tuple({A,B}),1) is simplified to B TEST_F(AlgebraicSimplifierTest, SelectMakeTuple) { Shape r0f32 = ShapeUtil::MakeShape(F32, {}); diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc index d3b94d7541..e57d49172b 100644 --- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc @@ -63,7 +63,7 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot, bool transpose_lhs, llvm::Value* executable_run_options_value, llvm::IRBuilder<>* ir_builder, const HloModuleConfig& hlo_module_config) { PrimitiveType type = target_array.GetShape().element_type(); - TF_RET_CHECK(F32 == type || F64 == type); + TF_RET_CHECK(F32 == type || F64 == type || C64 == type); DotOpEmitter dot_emitter(dot, transpose_lhs, transpose_rhs, target_array, lhs_array, rhs_array, executable_run_options_value, ir_builder, hlo_module_config); @@ -176,7 +176,7 @@ tensorflow::Status DotOpEmitter::Emit() { llvm::BasicBlock* preheader_bb = reduction_loop->GetPreheaderBasicBlock(); ir_builder_->SetInsertPoint(preheader_bb->getTerminator()); - ir_builder_->CreateStore(llvm::ConstantFP::get(accum_type, 0.0), + ir_builder_->CreateStore(llvm::Constant::getNullValue(accum_type), accum_address); // Body basic block of reduction loop: @@ -191,9 +191,29 @@ tensorflow::Status DotOpEmitter::Emit() { llvm::Value* rhs_element = rhs_array_.EmitReadArrayElement(rhs_index, ir_builder_); - llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element); llvm::Value* accum = ir_builder_->CreateLoad(accum_address); - llvm::Value* updated_accum = ir_builder_->CreateFAdd(accum, product); + llvm::Value* updated_accum; + if (ShapeUtil::ElementIsComplex(lhs_shape)) { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + llvm::Value* product_real = ir_builder_->CreateFSub( + ir_builder_->CreateFMul(real(lhs_element), real(rhs_element)), + ir_builder_->CreateFMul(imag(lhs_element), imag(rhs_element))); + llvm::Value* product_imag = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_element), imag(rhs_element)), + ir_builder_->CreateFMul(imag(lhs_element), real(rhs_element))); + updated_accum = ir_builder_->CreateInsertValue( + accum, ir_builder_->CreateFAdd(real(accum), product_real), {0}); + updated_accum = ir_builder_->CreateInsertValue( + updated_accum, ir_builder_->CreateFAdd(imag(accum), product_imag), {1}); + } else { + llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element); + updated_accum = ir_builder_->CreateFAdd(accum, product); + } ir_builder_->CreateStore(updated_accum, accum_address); // Exit basic block of reduction loop. @@ -230,11 +250,28 @@ tensorflow::Status DotOpEmitter::Emit() { tensorflow::Status DotOpEmitter::EmitScalarDot() { // A scalar dot is just a scalar multiply. + llvm::Value* result; llvm::Value* lhs_value = lhs_array_.EmitReadArrayElement(/*index=*/{}, ir_builder_); llvm::Value* rhs_value = rhs_array_.EmitReadArrayElement(/*index=*/{}, ir_builder_); - llvm::Value* result = ir_builder_->CreateFMul(lhs_value, rhs_value); + if (ShapeUtil::ElementIsComplex(lhs_array_.GetShape())) { +#define REAL(x) ir_builder_->CreateExtractValue(x, {0}) +#define IMAG(x) ir_builder_->CreateExtractValue(x, {1}) + llvm::Value* real = ir_builder_->CreateFSub( + ir_builder_->CreateFMul(REAL(lhs_value), REAL(rhs_value)), + ir_builder_->CreateFMul(IMAG(lhs_value), IMAG(rhs_value))); + llvm::Value* imag = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(REAL(lhs_value), IMAG(rhs_value)), + ir_builder_->CreateFMul(IMAG(lhs_value), REAL(rhs_value))); +#undef IMAG +#undef REAL + result = llvm::ConstantAggregateZero::get(lhs_array_.GetElementLlvmType()); + result = ir_builder_->CreateInsertValue(result, real, {0}); + result = ir_builder_->CreateInsertValue(result, imag, {1}); + } else { + result = ir_builder_->CreateFMul(lhs_value, rhs_value); + } target_array_.EmitWriteArrayElement(/*index=*/{}, result, ir_builder_); return tensorflow::Status::OK(); } diff --git a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc index 73e039250b..ba693ec89a 100644 --- a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc @@ -46,8 +46,8 @@ StatusOr CpuElementalIrEmitter::EmitFloatUnaryOp( } // Create function type for the function. llvm::FunctionType* function_type = llvm::FunctionType::get( - llvm_ir::PrimitiveTypeToIrType(element_type, ir_builder_), - llvm_ir::PrimitiveTypeToIrType(element_type, ir_builder_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), /*isVarArg=*/false); // Create function declaration for 'tanhf'. llvm::Function* function = diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc index ea5b6ca4eb..d72abede02 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc @@ -41,6 +41,12 @@ bool PotentiallyImplementedAsEigenConvolution( ShapeUtil::HasZeroElements(kernel_shape)) { return false; } + // TODO(b/65408531): Explore using Eigen dot for complex64 type. + if (ShapeUtil::ElementIsComplex(input_shape) || + ShapeUtil::ElementIsComplex(kernel_shape)) { + return false; + } + const ConvolutionDimensionNumbers& dnums = convolution.convolution_dimension_numbers(); // Only 1D and 2D convolutions are supported at the moment. diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 52085d1376..fa3b3ab8e7 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -288,7 +288,7 @@ Status IrEmitter::HandleConstant(HloInstruction* constant, MinimumAlignmentForShape(literal.shape())); } else { llvm::Constant* initializer = - llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_); + llvm_ir::ConvertLiteralToIrConstant(literal, module_); global_for_const = new llvm::GlobalVariable( /*Module=*/*module_, /*Type=*/initializer->getType(), @@ -401,7 +401,7 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, const Shape& shape = get_tuple_element->shape(); emitted_value_[get_tuple_element] = llvm_ir::EmitGetTupleElement( shape, get_tuple_element->tuple_index(), MinimumAlignmentForShape(shape), - GetEmittedValueFor(operand), &ir_builder_); + GetEmittedValueFor(operand), &ir_builder_, module_); return Status::OK(); } @@ -412,9 +412,9 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, if (ShapeUtil::IsTuple(select->shape())) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select)); - llvm_ir::EmitTupleSelect(GetIrArrayFor(select), GetIrArrayFor(pred), - GetEmittedValueFor(on_true), - GetEmittedValueFor(on_false), &ir_builder_); + llvm_ir::EmitTupleSelect( + GetIrArrayFor(select), GetIrArrayFor(pred), GetEmittedValueFor(on_true), + GetEmittedValueFor(on_false), &ir_builder_, module_); return Status::OK(); } @@ -459,7 +459,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) { tuple_element_addresses.push_back(tuple_element_address); } - llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_); + llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_, + module_); } else { TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kInfeed, shape, GetEmittedValueFor(infeed))); @@ -562,7 +563,7 @@ Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) { ShapeUtil::GetTupleElementShape(operand_shape, i); llvm::Value* tuple_element = llvm_ir::EmitGetTupleElement( tuple_element_shape, i, MinimumAlignmentForShape(tuple_element_shape), - value, &ir_builder_); + value, &ir_builder_, module_); TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kOutfeed, tuple_element_shape, tuple_element)); } @@ -583,7 +584,7 @@ Status IrEmitter::HandleTuple( for (auto operand : operands) { base_ptrs.push_back(GetEmittedValueFor(operand)); } - llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_); + llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_, module_); return Status::OK(); } @@ -644,7 +645,7 @@ Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window, // the initial value on the reduce_window. PrimitiveType operand_element_type = operand->shape().element_type(); llvm::Value* accumulator_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_), "reduce_window_accumulator_address", &ir_builder_, MinimumAlignmentForPrimitiveType(operand_element_type)); ir_builder_.CreateStore(ir_builder_.CreateLoad(GetEmittedValueFor( @@ -769,7 +770,7 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) { // Allocate space to keep the currently selected value, its index, and // the boolean initialized_flag, which is initially set to false. llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_), "selected_value_address", &ir_builder_, MinimumAlignmentForPrimitiveType(operand_element_type)); llvm::Value* selected_index_address = @@ -851,8 +852,8 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) { // If the 'select' function returns false, update the selected value and the // index to the currently visiting operand. llvm::Value* cond = ir_builder_.CreateICmpNE( - result, llvm::ConstantInt::get( - llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), 0), + result, + llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0), "boolean_predicate"); llvm_ir::LlvmIfData if_select_lhs = llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_); @@ -895,7 +896,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs, HloInstruction* rhs) { TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*dot, /*operands=*/{lhs, rhs}, - /*supported_types=*/{F32, F64})); + /*supported_types=*/{F32, F64, C64})); llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs)); llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs)); @@ -923,7 +924,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution, const Window& window) { TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*convolution, /*operands=*/{lhs, rhs}, - /*supported_types=*/{F32})); + /*supported_types=*/{F32, C64})); const ConvolutionDimensionNumbers& dnums = convolution->convolution_dimension_numbers(); @@ -1079,7 +1080,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution, // the output entry at the given index. PrimitiveType lhs_element_type = lhs->shape().element_type(); llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(lhs_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(lhs_element_type, module_), "convolution_sum_address", &ir_builder_, MinimumAlignmentForPrimitiveType(lhs_element_type)); ir_builder_.CreateStore( @@ -1295,14 +1296,14 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) { PrimitiveType element_type = operand->shape().element_type(); // Used to calculate E(X). llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), "sum_address", &ir_builder_, MinimumAlignmentForPrimitiveType(element_type)); // Used to calculate E(X^2). llvm::Value* sum_square_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), "sum_square_address", &ir_builder_, MinimumAlignmentForPrimitiveType(element_type)); @@ -1425,7 +1426,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) { .EmitLoop(IrName(batch_norm_training, "normalize"))); llvm_ir::EmitTuple(GetIrArrayFor(batch_norm_training), - {normalized, mean, var}, &ir_builder_); + {normalized, mean, var}, &ir_builder_, module_); return Status::OK(); } @@ -1488,6 +1489,14 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator( } const Shape& root_shape = root_instruction->shape(); + if (ShapeUtil::ElementIsComplex(root_shape)) { + // TODO(b/65408531): Complex add could by done via bitcast to + // Complex multiply would be more challenging. We could perhaps use a + // strided load to get all reals in a vector, all imags in a vector, or use + // CreateShuffleVector on a bitcast to float x [2N]. + *failure_reason = "complex values not supported"; + return nullptr; + } bool root_is_floating_point = ShapeUtil::ElementIsFloating(root_shape); bool root_is_integral = ShapeUtil::ElementIsIntegral(root_shape); bool root_is_signed = ShapeUtil::ElementIsSigned(root_shape); @@ -1509,7 +1518,7 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator( // This is visually similar to ElementalIrEmitter, though conceptually we're // doing something different here. ElementalIrEmitter emits scalar operations // while these emit scalar or vector operations depending on the type of the - // operands. + // operands. See CreateShardedVectorType for the actual types in use here. switch (root_instruction->opcode()) { default: *failure_reason = "did not recognize root instruction opcode"; @@ -1586,7 +1595,7 @@ IrEmitter::ShardedVectorType IrEmitter::CreateShardedVectorType( ShardedVectorType sharded_vector_type; llvm::Type* element_ir_type = - llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_); + llvm_ir::PrimitiveTypeToIrType(element_type, module_); for (int i = 0, e = 1 + tensorflow::Log2Ceiling(element_count); i < e; i++) { // For every power of two present in element_count, we generate one or more @@ -1919,7 +1928,7 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, // Initialize an accumulator with init_value. PrimitiveType accumulator_type = reduce->shape().element_type(); llvm::AllocaInst* accumulator_addr = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(accumulator_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(accumulator_type, module_), "accumulator", &ir_builder_, MinimumAlignmentForPrimitiveType(accumulator_type)); llvm::Value* init_value_addr = GetEmittedValueFor(init_value); @@ -2248,6 +2257,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { return Status::OK(); } else if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(fusion, assignment_)) { + VLOG(3) << "HandleFusion FusedDynamicUpdateSliceInPlace"; CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_); TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion)); @@ -2257,6 +2267,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { fusion, operands, GetIrArrayFor(fusion), &elemental_emitter, &ir_builder_); } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) { + VLOG(3) << "HandleFusion kLoop"; CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_); auto operands = GetIrArraysForOperandsOf(fusion); FusedIrEmitter fused_emitter(operands, &elemental_emitter); @@ -2400,8 +2411,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) { {while_result}, IrName(xla_while, "cond")); llvm::Value* while_predicate = ir_builder_.CreateICmpNE( while_condition, - llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), - 0)); + llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0)); // Branches to the body or to the while exit depending on the condition. llvm::BasicBlock* body_bb = llvm::BasicBlock::Create( @@ -2542,7 +2552,7 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source, unsigned element_alignment = GCD( primitive_type_size, MinimumAlignmentForPrimitiveType(primitive_type)); llvm::Type* primitive_ptr_type = llvm::PointerType::getUnqual( - llvm_ir::PrimitiveTypeToIrType(primitive_type, &ir_builder_)); + llvm_ir::PrimitiveTypeToIrType(primitive_type, module_)); if (element_count == 1) { auto* load_instruction = ir_builder_.CreateAlignedLoad( @@ -2755,7 +2765,7 @@ llvm::Value* IrEmitter::GetEmittedValueFor(const HloInstruction* hlo) { } llvm::Type* IrEmitter::IrShapeType(const Shape& shape) { - return llvm_ir::ShapeToIrType(shape, &ir_builder_); + return llvm_ir::ShapeToIrType(shape, module_); } std::vector IrEmitter::GetComputeFunctionParams() { @@ -2925,7 +2935,7 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall( PrimitiveType return_type = return_shape.element_type(); llvm::Value* return_value_buffer = llvm_ir::EmitAllocaAtFunctionEntryWithCount( - llvm_ir::PrimitiveTypeToIrType(return_type, &ir_builder_), elements, + llvm_ir::PrimitiveTypeToIrType(return_type, module_), elements, tensorflow::strings::StrCat(name, "_return_value_address"), &ir_builder_, MinimumAlignmentForPrimitiveType(return_type)); EmitArrayFunctionCallInto(function, parameter_addresses, return_value_buffer, @@ -3100,7 +3110,7 @@ Status IrEmitter::EmitTargetElementLoop( for (int64 i = 0; i < output_arrays.size(); ++i) { tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer()); } - llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_); + llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_, module_); } else { if (ShouldEmitParallelLoopFor(*target_op)) { diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h index 5b1dbf439c..adaff90913 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h @@ -85,6 +85,10 @@ class DfsHloVisitor { virtual Status HandleCopy(HloInstruction* copy) { return HandleElementwiseUnary(copy); } + virtual Status HandleComplex(HloInstruction* complex, HloInstruction* real, + HloInstruction* imag) { + return HandleElementwiseBinary(complex); + } virtual Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, HloInstruction* rhs) { return HandleElementwiseBinary(multiply); @@ -122,6 +126,10 @@ class DfsHloVisitor { virtual Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { return HandleElementwiseUnary(abs); } + virtual Status HandleAtan2(HloInstruction* atan2, HloInstruction* y, + HloInstruction* x) { + return HandleElementwiseBinary(atan2); + } virtual Status HandleRound(HloInstruction* round) { return HandleElementwiseUnary(round); } @@ -152,6 +160,12 @@ class DfsHloVisitor { virtual Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) { return HandleElementwiseUnary(tanh); } + virtual Status HandleReal(HloInstruction* real, HloInstruction* operand) { + return HandleElementwiseUnary(real); + } + virtual Status HandleImag(HloInstruction* imag, HloInstruction* operand) { + return HandleElementwiseUnary(imag); + } virtual Status HandleIsFinite(HloInstruction* is_finite, HloInstruction* operand) { return HandleElementwiseUnary(is_finite); diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc index 44f709bede..fd4c332cba 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc @@ -54,10 +54,12 @@ StatusOr ElementalIrEmitter::EmitUnaryOp( const HloInstruction* op, llvm::Value* operand_value) const { if (op->opcode() == HloOpcode::kCopy) { return operand_value; + } else if (operand_value->getType()->isIntegerTy()) { + return EmitIntegerUnaryOp(op, operand_value); + } else if (ShapeUtil::ElementIsComplex(op->operand(0)->shape())) { + return EmitComplexUnaryOp(op, operand_value); } else { - return operand_value->getType()->isIntegerTy() - ? EmitIntegerUnaryOp(op, operand_value) - : EmitFloatUnaryOp(op, operand_value); + return EmitFloatUnaryOp(op, operand_value); } } @@ -73,20 +75,35 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( } if (primitive_util::IsIntegralType(to_type)) { return ir_builder_->CreateIntCast( - operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_), + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_), primitive_util::IsSignedIntegralType(to_type)); } if (primitive_util::IsFloatingPointType(to_type)) { if (primitive_util::IsSignedIntegralType(from_type)) { return ir_builder_->CreateSIToFP( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } if (primitive_util::IsUnsignedIntegralType(from_type) || from_type == PRED) { return ir_builder_->CreateUIToFP( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); + } + } + if (primitive_util::IsComplexType(to_type)) { + auto to_ir_component_type = llvm_ir::PrimitiveTypeToIrType( + primitive_util::ComplexComponentType(to_type), module_); + if (primitive_util::IsSignedIntegralType(from_type)) { + return ComposeComplex( + op, + ir_builder_->CreateSIToFP(operand_value, to_ir_component_type), + nullptr); + } + if (primitive_util::IsUnsignedIntegralType(from_type) || + from_type == PRED) { + return ComposeComplex( + op, + ir_builder_->CreateUIToFP(operand_value, to_ir_component_type), + nullptr); } } return Unimplemented("conversion from primitive type %s to %s", @@ -97,8 +114,8 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( bool is_signed = primitive_util::IsSignedIntegralType(op->shape().element_type()); if (is_signed) { - auto type = llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), - ir_builder_); + auto type = + llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_); auto zero = llvm::ConstantInt::get(type, 0); auto cmp = ir_builder_->CreateICmpSGE(operand_value, zero); return ir_builder_->CreateSelect(cmp, operand_value, @@ -110,8 +127,8 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( case HloOpcode::kSign: { bool is_signed = primitive_util::IsSignedIntegralType(op->shape().element_type()); - auto type = llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), - ir_builder_); + auto type = + llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_); auto zero = llvm::ConstantInt::get(type, 0); auto cmp = ir_builder_->CreateICmpEQ(operand_value, zero); if (is_signed) { @@ -135,7 +152,7 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( return ir_builder_->CreateZExt( ir_builder_->CreateNot(ir_builder_->CreateTrunc( operand_value, ir_builder_->getInt1Ty())), - llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_)); + llvm_ir::PrimitiveTypeToIrType(PRED, module_)); } else if (primitive_util::IsIntegralType(type)) { return ir_builder_->CreateNot(operand_value); } @@ -157,20 +174,30 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( if (from_type == to_type) { return operand_value; } + if (primitive_util::IsComplexType(to_type)) { + PrimitiveType to_component_type = + primitive_util::ComplexComponentType(to_type); + if (from_type == to_component_type) { + return ComposeComplex(op, operand_value, nullptr); + } + return ComposeComplex( + op, + ir_builder_->CreateFPCast( + operand_value, + llvm_ir::PrimitiveTypeToIrType(to_component_type, module_)), + nullptr); + } if (primitive_util::IsFloatingPointType(to_type)) { return ir_builder_->CreateFPCast( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } if (primitive_util::IsSignedIntegralType(to_type)) { return ir_builder_->CreateFPToSI( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } if (primitive_util::IsUnsignedIntegralType(to_type)) { return ir_builder_->CreateFPToUI( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } return Unimplemented("unhandled conversion operation: %s => %s", PrimitiveType_Name(from_type).c_str(), @@ -230,7 +257,7 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( auto not_infinite = ir_builder_->CreateFCmpONE(abs_value, infinity); auto result_i1 = ir_builder_->CreateAnd(equal_self, not_infinite); return ir_builder_->CreateZExt( - result_i1, llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_)); + result_i1, llvm_ir::PrimitiveTypeToIrType(PRED, module_)); } case HloOpcode::kNegate: return ir_builder_->CreateFNeg(operand_value); @@ -240,20 +267,164 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( } } +StatusOr ElementalIrEmitter::EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + switch (op->opcode()) { + // TODO(b/65209142): Angle/Log require atan2. + // case HloOpcode::kAngle: + // case HloOpcode::kLog: // log(a+bi) = .5*log(a^2+b^2) + i*atan2(b, a) + case HloOpcode::kConvert: { + PrimitiveType from_type = op->operand(0)->shape().element_type(); + TF_RET_CHECK(primitive_util::IsComplexType(from_type)); + PrimitiveType to_type = op->shape().element_type(); + TF_RET_CHECK(primitive_util::IsComplexType(to_type)); + if (from_type == to_type) { + return operand_value; + } + PrimitiveType to_component_type = + primitive_util::ComplexComponentType(to_type); + auto to_ir_component_type = + llvm_ir::PrimitiveTypeToIrType(to_component_type, module_); + return ComposeComplex( + op, + ir_builder_->CreateFPCast(real(operand_value), to_ir_component_type), + ir_builder_->CreateFPCast(imag(operand_value), to_ir_component_type)); + } + case HloOpcode::kExp: { + // e^(a+bi) = e^a*(cos(b)+sin(b)i) + auto exp_a = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::exp, {real(operand_value)}, + {real(operand_value)->getType()}, ir_builder_); + auto cos_b = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::cos, {imag(operand_value)}, + {imag(operand_value)->getType()}, ir_builder_); + auto sin_b = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::sin, {imag(operand_value)}, + {imag(operand_value)->getType()}, ir_builder_); + return ComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b), + ir_builder_->CreateFMul(exp_a, sin_b)); + } + case HloOpcode::kCos: { + // cos(z) = .5(e^(iz) + e^(-iz)) + // cos(a+bi) = .5(e^(-b+ai) + e^(b-ai)) + // now, e^(x+yi) = e^x*(cos(y)+sin(y)i), so we have + // cos(a+bi) = .5(e^-b*(cos(a)+sin(a)i) + e^b*(cos(-a)+sin(-a)i)) + // cos(-x) = cos(x) and sin(-x) = -sin(x), so + // cos(a+bi) = .5(e^-b*(cos(a)+sin(a)i) + e^b*(cos(a)-sin(a)i)) + // = .5(cos(a)*(e^-b+e^b) + i*sin(a)*(e^-b-e^b)) + auto a = real(operand_value); + auto b = imag(operand_value); + auto type = a->getType(); + auto exp_b = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {b}, + {type}, ir_builder_); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b); + auto cos_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {a}, + {type}, ir_builder_); + auto sin_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {a}, + {type}, ir_builder_); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)), + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b))); + } + case HloOpcode::kSin: { + // sin(z) = .5i(e^(-iz) - e^(iz)) + // sin(a+bi) = .5i(e^(-i(a+bi)) - e^(i(a+bi))) + // = .5i(e^(b-ai) - e^(-b+ai)) + // now, e^(x+yi) = e^x*(cos(y)+sin(y)i), so we have + // sin(a+bi) = 0.5i(e^b*(cos(-a)+sin(-a)i) - e^-b*(cos(a)+sin(a)i)) + // = 0.5(e^b*(cos(-a)i-sin(-a)) - e^-b*(cos(a)i-sin(a))) + // cos(-x) = cos(x) and sin(-x) = -sin(x), so + // = 0.5(e^b*(cos(a)i+sin(a)) - e^-b*(cos(a)i-sin(a))) + // = 0.5(sin(a)*(e^b+e^-b) + i*cos(a)*(e^b-e^-b) + auto a = real(operand_value); + auto b = imag(operand_value); + auto type = a->getType(); + auto exp_b = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {b}, + {type}, ir_builder_); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b); + auto cos_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {a}, + {type}, ir_builder_); + auto sin_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {a}, + {type}, ir_builder_); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)), + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b))); + } + case HloOpcode::kAbs: { + auto sum_sq = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(operand_value), real(operand_value)), + ir_builder_->CreateFMul(imag(operand_value), imag(operand_value))); + return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sqrt, {sum_sq}, + {sum_sq->getType()}, ir_builder_); + } + case HloOpcode::kSign: { // Sign(c) = c / |c| + auto sum_sq = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(operand_value), real(operand_value)), + ir_builder_->CreateFMul(imag(operand_value), imag(operand_value))); + auto cplx_abs = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::sqrt, {sum_sq}, {sum_sq->getType()}, ir_builder_); + auto type = cplx_abs->getType(); + auto zero = llvm::ConstantFP::get(type, 0.0); + auto oeq = ir_builder_->CreateFCmpOEQ(cplx_abs, zero); + return ir_builder_->CreateSelect( + oeq, ComposeComplex(op, zero, zero), + ComposeComplex( + op, ir_builder_->CreateFDiv(real(operand_value), cplx_abs), + ir_builder_->CreateFDiv(imag(operand_value), cplx_abs))); + } + case HloOpcode::kNegate: + return ComposeComplex(op, ir_builder_->CreateFNeg(real(operand_value)), + ir_builder_->CreateFNeg(imag(operand_value))); + case HloOpcode::kReal: + return real(operand_value); + case HloOpcode::kImag: + return imag(operand_value); + default: + return Unimplemented("unary complex op '%s'", + HloOpcodeString(op->opcode()).c_str()); + } +} + StatusOr ElementalIrEmitter::EmitBinaryOp( const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const { - return lhs_value->getType()->isIntegerTy() - ? EmitIntegerBinaryOp(op, lhs_value, rhs_value, - primitive_util::IsSignedIntegralType( - op->operand(0)->shape().element_type())) - : EmitFloatBinaryOp(op, lhs_value, rhs_value); + PrimitiveType operand_type = op->operand(0)->shape().element_type(); + if (lhs_value->getType()->isIntegerTy()) { + return EmitIntegerBinaryOp( + op, lhs_value, rhs_value, + primitive_util::IsSignedIntegralType(operand_type)); + } else if (primitive_util::IsComplexType(operand_type)) { + return EmitComplexBinaryOp(op, lhs_value, rhs_value); + } else { + return EmitFloatBinaryOp(op, lhs_value, rhs_value); + } } StatusOr ElementalIrEmitter::EmitFloatBinaryOp( const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const { switch (op->opcode()) { + // case HloOpcode::kAtan2: // TODO(b/65209142): CPU atan2 support + case HloOpcode::kComplex: + return ComposeComplex(op, lhs_value, rhs_value); case HloOpcode::kAdd: return ir_builder_->CreateFAdd(lhs_value, rhs_value); case HloOpcode::kSubtract: @@ -305,6 +476,88 @@ StatusOr ElementalIrEmitter::EmitFloatBinaryOp( } } +StatusOr ElementalIrEmitter::EmitComplexBinaryOp( + const HloInstruction* op, llvm::Value* lhs_value, + llvm::Value* rhs_value) const { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + switch (op->opcode()) { + case HloOpcode::kAdd: + return ComposeComplex( + op, ir_builder_->CreateFAdd(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFAdd(imag(lhs_value), imag(rhs_value))); + case HloOpcode::kSubtract: + return ComposeComplex( + op, ir_builder_->CreateFSub(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFSub(imag(lhs_value), imag(rhs_value))); + case HloOpcode::kMultiply: + return ComposeComplex( + op, + ir_builder_->CreateFSub( + ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), imag(rhs_value))), + ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_value), imag(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value)))); + case HloOpcode::kDivide: { + // (a+bi) / (c+di) = ((a+bi)(c-di)) / ((c+di)(c-di)) + // = ((ac + bd) + (bc - ad)i) / (c^2 + d^2) + auto rhs_sum_sq = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(rhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(rhs_value), imag(rhs_value))); + auto type = rhs_sum_sq->getType(); + auto zero = llvm::ConstantFP::get(type, 0.0); + auto oeq = ir_builder_->CreateFCmpOEQ(rhs_sum_sq, zero); + return ir_builder_->CreateSelect( + oeq, ComposeComplex(op, llvm::ConstantFP::getInfinity(type), zero), + ComposeComplex( + op, + ir_builder_->CreateFDiv( + ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), + imag(rhs_value))), + rhs_sum_sq), + ir_builder_->CreateFDiv( + ir_builder_->CreateFSub( + ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(real(lhs_value), + imag(rhs_value))), + rhs_sum_sq))); + } + // LLVM comparisons can be "unordered" (U) or "ordered" (O) -- ordered + // comparisons always return false when one of the operands is NaN, whereas + // unordered comparisons return true. + // + // We use ordered comparisons for everything except kNe, where we use an + // unordered comparison. This makes x != y equivalent to !(x == y), and + // matches C++'s semantics. + case HloOpcode::kEq: + return ir_builder_->CreateAnd( + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, real(lhs_value), + real(rhs_value), ir_builder_), + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, imag(lhs_value), + imag(rhs_value), ir_builder_)); + case HloOpcode::kNe: + return ir_builder_->CreateOr( + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, real(lhs_value), + real(rhs_value), ir_builder_), + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, imag(lhs_value), + imag(rhs_value), ir_builder_)); + + // TODO(b/65209142): requires arg(z) -> requires atan|atan2 intrinsic + // case HloOpcode::kPower: + // // (a+bi)^(c+di) = exp(i(c+di)*arg(a+bi)) * (a*a+b*b)^(c/2+di/2) + default: + return Unimplemented("binary complex op '%s'", + HloOpcodeString(op->opcode()).c_str()); + } +} + llvm::Value* ElementalIrEmitter::EmitFloatMax(llvm::Value* lhs_value, llvm::Value* rhs_value) const { return llvm_ir::EmitFloatMax(lhs_value, rhs_value, ir_builder_); @@ -396,7 +649,7 @@ StatusOr ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type, StatusOr ElementalIrEmitter::EmitErfcInv( PrimitiveType prim_type, llvm::Value* value) const { // Compute erfcinv(value) by calculating erfinv(1.0 - value). - auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, ir_builder_); + auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, module_); auto one = llvm::ConstantFP::get(type, 1.0); return EmitErfInv(prim_type, ir_builder_->CreateFSub(one, value)); } @@ -619,7 +872,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator( const { PrimitiveType param_prim_type = hlo->operand(0)->shape().element_type(); llvm::Type* param_ir_type = - llvm_ir::PrimitiveTypeToIrType(param_prim_type, ir_builder_); + llvm_ir::PrimitiveTypeToIrType(param_prim_type, module_); // Same values as PCG library // https://github.com/imneme/pcg-c/blob/master/include/pcg_variants.h @@ -783,7 +1036,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator( return ir_builder_->CreateZExt( ir_builder_->CreateFCmpOLT(get_next_uniform_float(), p), llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_)); + module_)); } default: return InvalidArgument( @@ -806,9 +1059,11 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( case HloOpcode::kCos: case HloOpcode::kExp: case HloOpcode::kFloor: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLog: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kTanh: @@ -821,6 +1076,8 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( return EmitUnaryOp(hlo, operand_value); }; case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kEq: case HloOpcode::kGe: @@ -913,10 +1170,10 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( } llvm_ir::SetToFirstInsertPoint(exit_block, ir_builder_); - llvm::PHINode* output = ir_builder_->CreatePHI( - llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_), - hlo->operands().size()); + llvm::PHINode* output = + ir_builder_->CreatePHI(llvm_ir::PrimitiveTypeToIrType( + hlo->shape().element_type(), module_), + hlo->operands().size()); auto prior_insert_point = ir_builder_->GetInsertPoint(); ir_builder_->SetInsertPoint(init_block); @@ -1075,7 +1332,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( // else -> return data from 'index'. llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry( llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_), + module_), "ret_value_addr", ir_builder_); llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse( slice_intersection, "slice_intersection", ir_builder_); @@ -1164,7 +1421,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( // } llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry( llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_), + module_), "pad_result_addr", ir_builder_); llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", ir_builder_); @@ -1206,7 +1463,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( ir_builder_); PrimitiveType primitive_type = hlo->shape().element_type(); llvm::Type* primitive_type_llvm = - llvm_ir::PrimitiveTypeToIrType(primitive_type, ir_builder_); + llvm_ir::PrimitiveTypeToIrType(primitive_type, module_); llvm::Value* accumulator_alloca = llvm_ir::EmitAllocaAtFunctionEntry( primitive_type_llvm, "dot_acc", ir_builder_); ir_builder_->CreateStore( @@ -1239,7 +1496,28 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( TF_ASSIGN_OR_RETURN(llvm::Value * lhs_value, lhs_generator(lhs_index)); TF_ASSIGN_OR_RETURN(llvm::Value * rhs_value, rhs_generator(rhs_index)); llvm::Value* next_accumulator; - if (primitive_util::IsFloatingPointType(primitive_type)) { + if (primitive_util::IsComplexType(primitive_type)) { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + llvm::Value* product_real = ir_builder_->CreateFSub( + ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), imag(rhs_value))); + llvm::Value* product_imag = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_value), imag(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value))); + next_accumulator = ir_builder_->CreateInsertValue( + current_accumulator, + ir_builder_->CreateFAdd(real(current_accumulator), product_real), + {0}); + next_accumulator = ir_builder_->CreateInsertValue( + next_accumulator, + ir_builder_->CreateFAdd(imag(current_accumulator), product_imag), + {1}); + } else if (primitive_util::IsFloatingPointType(primitive_type)) { next_accumulator = ir_builder_->CreateFAdd( current_accumulator, ir_builder_->CreateFMul(lhs_value, rhs_value)); @@ -1261,4 +1539,17 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( } } +llvm::Value* ElementalIrEmitter::ComposeComplex(const HloInstruction* op, + llvm::Value* real, + llvm::Value* imag) const { + auto cplx_type = + llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_); + auto complex = ir_builder_->CreateInsertValue( + llvm::ConstantAggregateZero::get(cplx_type), real, {0}); + if (imag != nullptr) { + complex = ir_builder_->CreateInsertValue(complex, imag, {1}); + } + return complex; +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/elemental_ir_emitter.h index 35dfa88e9b..9d32436e38 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.h +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.h @@ -55,6 +55,7 @@ class ElementalIrEmitter { const HloToElementGeneratorMap& operand_to_generator) const; llvm::IRBuilder<>* ir_builder() const { return ir_builder_; } + llvm::Module* module() const { return module_; } protected: virtual StatusOr EmitIntegerUnaryOp( @@ -63,6 +64,9 @@ class ElementalIrEmitter { virtual StatusOr EmitFloatUnaryOp( const HloInstruction* op, llvm::Value* operand_value) const; + virtual StatusOr EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const; + virtual StatusOr EmitIntegerBinaryOp(const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value, @@ -72,6 +76,10 @@ class ElementalIrEmitter { const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const; + virtual StatusOr EmitComplexBinaryOp( + const HloInstruction* op, llvm::Value* lhs_value, + llvm::Value* rhs_value) const; + virtual llvm::Value* EmitFloatMax(llvm::Value* lhs_value, llvm::Value* rhs_value) const; @@ -109,6 +117,11 @@ class ElementalIrEmitter { // compiled executable outside of the HLO code itself. const HloModuleConfig& hlo_module_config_; + protected: + // Composes a complex struct. imag may be nullptr for simple cast operations. + llvm::Value* ComposeComplex(const HloInstruction* op, llvm::Value* real, + llvm::Value* imag) const; + private: // Returns a ElementGenerator for a RNG HloInstruction. llvm_ir::ElementGenerator MakeRngElementGenerator( diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc index 8810a85cee..1b94499bc6 100644 --- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc @@ -135,6 +135,10 @@ StatusOr GpuElementalIrEmitter::EmitFloatBinaryOp( PrimitiveType rhs_input_type = op->operand(1)->shape().element_type(); PrimitiveType output_type = op->shape().element_type(); switch (op->opcode()) { + case HloOpcode::kAtan2: + return EmitLibdeviceMathCall("__nv_atan2", {lhs_value, rhs_value}, + {lhs_input_type, rhs_input_type}, + output_type); case HloOpcode::kRemainder: { return EmitLibdeviceMathCall("__nv_fmod", {lhs_value, rhs_value}, {lhs_input_type, rhs_input_type}, @@ -226,6 +230,112 @@ StatusOr GpuElementalIrEmitter::EmitFloatUnaryOp( } } +StatusOr GpuElementalIrEmitter::EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const { + PrimitiveType input_type = op->operand(0)->shape().element_type(); + PrimitiveType component_type = + primitive_util::IsComplexType(input_type) + ? primitive_util::ComplexComponentType(input_type) + : input_type; + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + + switch (op->opcode()) { + case HloOpcode::kLog: { + // log(a+bi) = .5*log(a^2+b^2) + i*atan2(b, a) + auto a = real(operand_value); + auto b = imag(operand_value); + llvm::Type* llvm_ty = a->getType(); + auto sum_sq = ir_builder_->CreateFAdd(ir_builder_->CreateFMul(a, a), + ir_builder_->CreateFMul(b, b)); + TF_ASSIGN_OR_RETURN( + auto log_sum_sq, + EmitLibdeviceMathCall("__nv_log", {sum_sq}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto angle, EmitLibdeviceMathCall("__nv_atan2", {b, a}, + {component_type, component_type}, + component_type)); + auto one_half = llvm::ConstantFP::get(llvm_ty, 0.5); + return ComposeComplex(op, ir_builder_->CreateFMul(one_half, log_sum_sq), + angle); + } + // TODO(b/65408531): Implement kPower on GPU, where atan2 is available. + // case HloOpcode::kPower: + // // (a+bi)^(c+di) = exp(i(c+di)*arg(a+bi)) * (a*a+b*b)^(0.5(c+di)) + case HloOpcode::kExp: { + // e^(a+bi) = e^a*(cos(b)+sin(b)i) + auto b = imag(operand_value); + TF_ASSIGN_OR_RETURN( + auto exp_a, EmitLibdeviceMathCall("__nv_exp", {real(operand_value)}, + {component_type}, component_type)); + TF_ASSIGN_OR_RETURN( + auto cos_b, EmitLibdeviceMathCall("__nv_cos", {b}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto sin_b, EmitLibdeviceMathCall("__nv_sin", {b}, {component_type}, + component_type)); + return ComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b), + ir_builder_->CreateFMul(exp_a, sin_b)); + } + case HloOpcode::kCos: { + // cos(a+bi) = .5(cos(a)*(e^-b+e^b) + i*sin(a)*(e^-b-e^b)) + auto a = real(operand_value); + auto llvm_ty = a->getType(); + TF_ASSIGN_OR_RETURN( + auto exp_b, EmitLibdeviceMathCall("__nv_exp", {imag(operand_value)}, + {component_type}, component_type)); + TF_ASSIGN_OR_RETURN( + auto cos_a, EmitLibdeviceMathCall("__nv_cos", {a}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto sin_a, EmitLibdeviceMathCall("__nv_sin", {a}, {component_type}, + component_type)); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)), + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b))); + } + + case HloOpcode::kSin: { + // sin(a+bi) = 0.5(sin(a)*(e^b+e^-b) + i*cos(a)*(e^b-e^-b) + auto a = real(operand_value); + auto llvm_ty = a->getType(); + TF_ASSIGN_OR_RETURN( + auto exp_b, EmitLibdeviceMathCall("__nv_exp", {imag(operand_value)}, + {component_type}, component_type)); + TF_ASSIGN_OR_RETURN( + auto cos_a, EmitLibdeviceMathCall("__nv_cos", {a}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto sin_a, EmitLibdeviceMathCall("__nv_sin", {a}, {component_type}, + component_type)); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)), + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b))); + } + default: + return ElementalIrEmitter::EmitComplexUnaryOp(op, operand_value); + } +} + llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall( const string& callee_name, tensorflow::gtl::ArraySlice operands, @@ -235,13 +345,12 @@ llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall( std::vector ir_input_types; for (PrimitiveType input_type : input_types) { ir_input_types.push_back( - llvm_ir::PrimitiveTypeToIrType(input_type, ir_builder_)); + llvm_ir::PrimitiveTypeToIrType(input_type, module_)); } llvm::FunctionType* callee_type = llvm::FunctionType::get( - llvm_ir::PrimitiveTypeToIrType(output_type, - ir_builder_), // The return type. - ir_input_types, // The parameter types. - false); // No variadic arguments. + llvm_ir::PrimitiveTypeToIrType(output_type, module_), // Return type. + ir_input_types, // Parameter types. + false); // No variadic arguments. // Declares the callee if it is not declared already. llvm::Function* callee = llvm::cast( @@ -315,7 +424,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator( PrimitiveType operand_element_type = operand->shape().element_type(); llvm::Value* accum_ptr = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_), "reduce_window_accum_ptr", ir_builder_); { TF_ASSIGN_OR_RETURN(llvm::Value * init_value, @@ -377,7 +486,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator( const HloInstruction* operand = hlo->operand(0); llvm::Value* accum_ptr = ir_builder()->CreateAlloca(llvm_ir::PrimitiveTypeToIrType( - hlo->shape().element_type(), ir_builder())); + hlo->shape().element_type(), module_)); TF_ASSIGN_OR_RETURN(llvm::Value * init_value, operand_to_generator.at(hlo->operand(1))({})); ir_builder()->CreateStore(init_value, accum_ptr); diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h index 6ddfc3710c..3defa1b696 100644 --- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h @@ -54,6 +54,9 @@ class GpuElementalIrEmitter : public ElementalIrEmitter { StatusOr EmitFloatUnaryOp( const HloInstruction* op, llvm::Value* operand_value) const override; + StatusOr EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const override; + StatusOr EmitFloatBinaryOp( const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const override; diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index 152d226ab0..163a161353 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -102,7 +102,7 @@ void HloToIrBindings::EmitBasePointersForHlos( slice_result.ConsumeValueOrDie(); if (slice.allocation()->is_thread_local()) { llvm::Type* pointee_type = - llvm_ir::ShapeToIrType(non_io_hlo->shape(), ir_builder_); + llvm_ir::ShapeToIrType(non_io_hlo->shape(), module_); BindHloToIrValue(*non_io_hlo, ir_builder_->CreateAlloca(pointee_type), index); } else { @@ -124,18 +124,18 @@ llvm::Value* HloToIrBindings::EmitGetTupleElement(const HloInstruction* gte, if (gte->operand(0)->opcode() != HloOpcode::kGetTupleElement) { return llvm_ir::EmitGetTupleElement( gte->shape(), gte->tuple_index(), /*alignment=*/1, - GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_); + GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_, module_); } return llvm_ir::EmitGetTupleElement( gte->shape(), gte->tuple_index(), /*alignment=*/1, - EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_); + EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_, module_); } llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, const ShapeIndex& shape_index, llvm::Value* ir_value) { llvm::Type* pointee_type = llvm_ir::ShapeToIrType( - ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_builder_); + ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_); llvm::Type* dest_type = pointee_type->getPointerTo(); llvm::Value* typed_ir_value; diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h index d43e09e8a8..a3120f15bc 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h @@ -36,10 +36,12 @@ class HloToIrBindings { public: HloToIrBindings(const HloModule& module, const BufferAssignment* buffer_assignment, - llvm::IRBuilder<>* ir_builder, bool is_nested) + llvm::IRBuilder<>* ir_builder, llvm::Module* llvm_module, + bool is_nested) : buffer_assignment_(buffer_assignment), is_nested_(is_nested), ir_builder_(ir_builder), + module_(llvm_module), alias_analysis_(module, *buffer_assignment_, &ir_builder_->getContext()) {} @@ -93,6 +95,7 @@ class HloToIrBindings { const bool is_nested_; llvm::IRBuilder<>* ir_builder_; + llvm::Module* module_; // Stores the underlying llvm::IrArray for each HloInstruction. // For an instruction that generates multiple outputs, the root will be a diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc index 3862c2190b..23765e05e8 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc @@ -53,9 +53,10 @@ namespace gpu { IrEmitter::IrEmitter(const HloModuleConfig& hlo_module_config, IrEmitterContext* ir_emitter_context, bool is_nested) : ir_emitter_context_(ir_emitter_context), - ir_builder_(ir_emitter_context->llvm_module()->getContext()), + module_(ir_emitter_context->llvm_module()), + ir_builder_(module_->getContext()), bindings_(ir_emitter_context->hlo_module(), - &ir_emitter_context->buffer_assignment(), &ir_builder_, + &ir_emitter_context->buffer_assignment(), &ir_builder_, module_, is_nested), hlo_module_config_(hlo_module_config) { ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags( @@ -71,18 +72,17 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) { }; } return EmitTargetElementLoop( - *hlo, GpuElementalIrEmitter(hlo_module_config_, - ir_emitter_context_->llvm_module(), - &ir_builder_, GetNestedComputer()) + *hlo, GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_, + GetNestedComputer()) .MakeElementGenerator(hlo, operand_to_generator)); } Status IrEmitter::HandleConstant(HloInstruction* constant, const Literal& literal) { llvm::Constant* initializer = - llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_); + llvm_ir::ConvertLiteralToIrConstant(literal, module_); llvm::GlobalVariable* global_for_const = new llvm::GlobalVariable( - *ir_emitter_context_->llvm_module(), initializer->getType(), + *module_, initializer->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, initializer, /*Name=*/""); VLOG(2) << "HandleConstant: " << constant->ToString() << std::endl @@ -115,7 +115,7 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, get_tuple_element->shape(), get_tuple_element->tuple_index(), // TODO(b/26344050): tighten the alignment here // based on the real element type. - /*alignment=*/1, GetBasePointer(*operand), &ir_builder_)); + /*alignment=*/1, GetBasePointer(*operand), &ir_builder_, module_)); return Status::OK(); } @@ -140,7 +140,7 @@ Status IrEmitter::HandleTuple( for (const HloInstruction* operand : operands) { base_ptrs.push_back(GetBasePointer(*operand)); } - llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_); + llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_, module_); return Status::OK(); } @@ -329,7 +329,7 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, if (ShapeUtil::IsTuple(select->shape())) { llvm_ir::EmitTupleSelect(GetIrArray(*select), GetIrArray(*pred), GetBasePointer(*on_true), - GetBasePointer(*on_false), &ir_builder_); + GetBasePointer(*on_false), &ir_builder_, module_); return Status::OK(); } @@ -355,7 +355,26 @@ Status IrEmitter::HandleDot(HloInstruction* dot, lhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_); llvm::Value* rhs_value = rhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_); - llvm::Value* result = ir_builder_.CreateFMul(lhs_value, rhs_value); + llvm::Value* result; + if (ShapeUtil::ElementIsComplex(lhs_shape)) { + auto real = [&](llvm::Value* x) { + return ir_builder_.CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_.CreateExtractValue(x, {1}); + }; + llvm::Value* real_result = ir_builder_.CreateFSub( + ir_builder_.CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_.CreateFMul(imag(lhs_value), imag(rhs_value))); + llvm::Value* imag_result = ir_builder_.CreateFAdd( + ir_builder_.CreateFMul(real(lhs_value), imag(rhs_value)), + ir_builder_.CreateFMul(imag(lhs_value), real(rhs_value))); + result = llvm::ConstantAggregateZero::get(lhs_array.GetElementLlvmType()); + result = ir_builder_.CreateInsertValue(result, real_result, {0}); + result = ir_builder_.CreateInsertValue(result, imag_result, {1}); + } else { + result = ir_builder_.CreateFMul(lhs_value, rhs_value); + } target_array.EmitWriteArrayElement(/*index=*/{}, result, &ir_builder_); return Status::OK(); } @@ -411,8 +430,8 @@ Status IrEmitter::HandleDot(HloInstruction* dot, // Initialize the accumulator in the preheader to zero. new llvm::StoreInst( - llvm::ConstantFP::get(accum_type, 0.0), // The value stored. - accum_address, // The address. + llvm::Constant::getNullValue(lhs_array.GetElementLlvmType()), // init 0 + accum_address, // The address. reduction_loop->GetPreheaderBasicBlock() ->getTerminator()); // The instruction this store is inserted before. @@ -427,9 +446,27 @@ Status IrEmitter::HandleDot(HloInstruction* dot, lhs_array.EmitReadArrayElement(lhs_index, &ir_builder_); llvm::Value* rhs_element = rhs_array.EmitReadArrayElement(rhs_index, &ir_builder_); - llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element); llvm::Value* accum = ir_builder_.CreateLoad(accum_address); - llvm::Value* updated_accum = ir_builder_.CreateFAdd(accum, product); + llvm::Value* updated_accum; + if (ShapeUtil::ElementIsComplex(lhs_shape)) { +#define REAL(x) ir_builder_.CreateExtractValue(x, {0}) +#define IMAG(x) ir_builder_.CreateExtractValue(x, {1}) + llvm::Value* product_real = ir_builder_.CreateFSub( + ir_builder_.CreateFMul(REAL(lhs_element), REAL(rhs_element)), + ir_builder_.CreateFMul(IMAG(lhs_element), IMAG(rhs_element))); + llvm::Value* product_imag = ir_builder_.CreateFAdd( + ir_builder_.CreateFMul(REAL(lhs_element), IMAG(rhs_element)), + ir_builder_.CreateFMul(IMAG(lhs_element), REAL(rhs_element))); + updated_accum = ir_builder_.CreateInsertValue( + accum, ir_builder_.CreateFAdd(REAL(accum), product_real), {0}); + updated_accum = ir_builder_.CreateInsertValue( + updated_accum, ir_builder_.CreateFAdd(IMAG(accum), product_imag), {1}); +#undef IMAG +#undef REAL + } else { + llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element); + updated_accum = ir_builder_.CreateFAdd(accum, product); + } ir_builder_.CreateStore(updated_accum, accum_address); // After the reduction loop exits, store the accumulator into the target @@ -494,7 +531,7 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, // Initialize an accumulator with init_value. llvm::AllocaInst* accumulator_addr = ir_builder_.CreateAlloca(llvm_ir::PrimitiveTypeToIrType( - reduce->shape().element_type(), &ir_builder_)); + reduce->shape().element_type(), module_)); ir_builder_.CreateStore( ir_builder_.CreateLoad(GetBasePointer(*init_value)), accumulator_addr); @@ -547,8 +584,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { for (HloInstruction* operand : fusion->operands()) { parameter_arrays.push_back(GetIrArray(*operand)); } - GpuElementalIrEmitter elemental_emitter(hlo_module_config_, - ir_emitter_context_->llvm_module(), + GpuElementalIrEmitter elemental_emitter(hlo_module_config_, module_, &ir_builder_, GetNestedComputer()); FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter); TF_RETURN_IF_ERROR(fusion->fused_expression_root()->Accept(&fused_emitter)); @@ -591,9 +627,8 @@ Status IrEmitter::HandleRng(HloInstruction* random, // Emits a single-threaded loop because the loop body generated by the element // generator for Rng can't be parallelized (b/32333178). return llvm_ir::LoopEmitter( - GpuElementalIrEmitter(hlo_module_config_, - ir_emitter_context_->llvm_module(), - &ir_builder_, GetNestedComputer()) + GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_, + GetNestedComputer()) .MakeElementGenerator(random, operand_to_generator), GetIrArray(*random), &ir_builder_) .EmitLoop(IrName(random)); @@ -634,7 +669,7 @@ StatusOr IrEmitter::ComputeNestedElement( tensorflow::gtl::ArraySlice parameter_elements) { llvm::Value* return_buffer = llvm_ir::EmitAllocaAtFunctionEntry( llvm_ir::PrimitiveTypeToIrType( - computation.root_instruction()->shape().element_type(), &ir_builder_), + computation.root_instruction()->shape().element_type(), module_), "return_buffer", &ir_builder_); std::vector parameter_buffers; for (llvm::Value* parameter_element : parameter_elements) { diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h index 29c3761dc5..90f40639d5 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h @@ -162,6 +162,7 @@ class IrEmitter : public DfsHloVisitorWithDefault { } IrEmitterContext* ir_emitter_context_; + llvm::Module* module_; // The following fields track the IR emission state. According to LLVM memory // management rules, their memory is owned by the module. diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc index 57f010530c..5da1a130d5 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc @@ -52,9 +52,9 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation( io_hlos->push_back(param); const Shape& param_shape = param->shape(); argument_types.push_back( - llvm_ir::ShapeToIrType(param_shape, &ir_builder_)->getPointerTo()); - int64 param_size = llvm_ir::ByteSizeOf( - param_shape, ir_emitter_context_->llvm_module()->getDataLayout()); + llvm_ir::ShapeToIrType(param_shape, module_)->getPointerTo()); + int64 param_size = + llvm_ir::ByteSizeOf(param_shape, module_->getDataLayout()); argument_dereferenceable_bytes.push_back(param_size); } { @@ -62,7 +62,7 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation( io_hlos->push_back(root); const Shape& root_shape = root->shape(); argument_types.push_back( - llvm_ir::ShapeToIrType(root_shape, &ir_builder_)->getPointerTo()); + llvm_ir::ShapeToIrType(root_shape, module_)->getPointerTo()); int64 root_size = llvm_ir::ByteSizeOf( root_shape, ir_emitter_context_->llvm_module()->getDataLayout()); argument_dereferenceable_bytes.push_back(root_size); diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 734c793c15..1c7e18304d 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -757,8 +757,8 @@ Status IrEmitterUnnested::EmitColumnReduction( auto loop_body_emitter = [=](const llvm_ir::IrArray::Index& tile_index) -> Status { // Emit the loop body that reduces one tile. - llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType( - input_shape.element_type(), &ir_builder_); + llvm::Type* element_ir_type = + llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_); llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca( element_ir_type, /*ArraySize=*/nullptr, "partial_reduction_result"); { @@ -973,7 +973,7 @@ Status IrEmitterUnnested::EmitRowReduction( [=](const llvm_ir::IrArray::Index& tile_index) -> Status { // Emit the loop body that reduces one tile. llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType( - input_shape.element_type(), &ir_builder_); + input_shape.element_type(), ir_emitter_context_->llvm_module()); llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca( element_ir_type, /*ArraySize=*/nullptr, "partial_reduction_result"); { @@ -1360,7 +1360,8 @@ Status IrEmitterUnnested::HandleSelectAndScatter( // boolean flag if the value is initialized. The initialized_flag is set // false. llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, + ir_emitter_context_->llvm_module()), "selected_value_address", &ir_builder_); llvm::Value* selected_index_address = llvm_ir::EmitAllocaAtFunctionEntryWithCount( @@ -1440,7 +1441,8 @@ Status IrEmitterUnnested::HandleSelectAndScatter( llvm::Value* operand_address = operand_array.EmitArrayElementAddress(operand_index, &ir_builder_); llvm::Value* select_return_buffer = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(PRED, + ir_emitter_context_->llvm_module()), "select_return_buffer", &ir_builder_); TF_RETURN_IF_ERROR(EmitCallToNestedComputation( *select_and_scatter->select(), @@ -1450,8 +1452,10 @@ Status IrEmitterUnnested::HandleSelectAndScatter( // If the 'select' function returns false, update the selected value and the // index to the currently visiting operand. llvm::Value* cond = ir_builder_.CreateICmpNE( - result, llvm::ConstantInt::get( - llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), 0), + result, + llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType( + PRED, ir_emitter_context_->llvm_module()), + 0), "boolean_predicate"); llvm_ir::LlvmIfData if_select_lhs = llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_); @@ -1877,7 +1881,8 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk( tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer()); } ir_builder_.SetInsertPoint(ir_builder_.GetInsertBlock()->getTerminator()); - llvm_ir::EmitTuple(GetIrArray(hlo), tuple_operand_ptrs, &ir_builder_); + llvm_ir::EmitTuple(GetIrArray(hlo), tuple_operand_ptrs, &ir_builder_, + module_); return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index fa6a8f3d53..f4a2c3d0e8 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -50,6 +50,12 @@ namespace xla { namespace { +template +struct is_complex_t : public std::false_type {}; + +template <> +struct is_complex_t : public std::true_type {}; + template StatusOr> Compare(const Shape& shape, HloOpcode opcode, const Literal& lhs_literal, @@ -101,6 +107,37 @@ StatusOr> Compare(const Shape& shape, HloOpcode opcode, return std::move(result); } +template <> +StatusOr> Compare( + const Shape& shape, HloOpcode opcode, const Literal& lhs_literal, + const Literal& rhs_literal) { + std::function compare_op; + switch (opcode) { + case HloOpcode::kEq: + compare_op = [](complex64 lhs_el, complex64 rhs_el) { + return lhs_el == rhs_el; + }; + break; + case HloOpcode::kNe: + compare_op = [](complex64 lhs_el, complex64 rhs_el) { + return lhs_el != rhs_el; + }; + break; + default: + LOG(FATAL) << "unhandled HLO opcode for conversion to Comparison: " + << HloOpcodeString(opcode); + } + + auto result = Literal::CreateFromShape(shape); + TF_RETURN_IF_ERROR(result->Populate( + [&](tensorflow::gtl::ArraySlice multi_index) { + return compare_op(lhs_literal.Get(multi_index), + rhs_literal.Get(multi_index)); + })); + + return std::move(result); +} + template StatusOr> ElementWiseUnaryOpImpl( HloInstruction* instruction, @@ -138,7 +175,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { Status DefaultAction(HloInstruction* hlo_instruction) override { return Unimplemented("unhandled HLO ops for HloEvaluator: %s.", HloOpcodeString(hlo_instruction->opcode()).c_str()); - }; + } // TODO(b/35950897): many of the stl functions used in the handlers are not // overloaded for every XLA primitive types. @@ -156,7 +193,8 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { template < typename NativeT, - typename std::enable_if::value>::type* = nullptr> + typename std::enable_if::value || + is_complex_t::value>::type* = nullptr> Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[abs], ElementWiseUnaryOp(abs, [](NativeT elem_operand) { @@ -169,7 +207,10 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return HandleAbs(abs, operand); } - Status HandleRound(HloInstruction* round) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRound(HloInstruction* round) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[round], ElementWiseUnaryOp(round, [](ReturnT elem_operand) { return std::round(elem_operand); @@ -177,6 +218,17 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRound(HloInstruction* round) { + return InvalidArgument("Unsupported type for Round"); + } + + Status HandleRound(HloInstruction* round) override { + return HandleRound(round); + } + Status HandleBroadcast(HloInstruction* broadcast) override { parent_->evaluated_[broadcast] = Literal::CreateFromShape(broadcast->shape()); @@ -205,15 +257,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } return operand_to_broadcast.Get(broadcast_indices); }); - }; + } - Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleCeil(HloInstruction* ceil) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[ceil], ElementWiseUnaryOp(ceil, [](ReturnT elem_operand) { return std::ceil(elem_operand); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleCeil(HloInstruction* ceil) { + return InvalidArgument("Unsupported type for Ceil"); + } + + Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) override { + return HandleCeil(ceil); + } Status HandleConvert(HloInstruction* convert) override { const HloInstruction* operand = convert->operand(0); @@ -237,15 +303,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return std::exp(elem_operand); })); return Status::OK(); - }; + } - Status HandleFloor(HloInstruction* floor, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleFloor(HloInstruction* floor) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[floor], ElementWiseUnaryOp(floor, [](ReturnT elem_operand) { return std::floor(elem_operand); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleFloor(HloInstruction* floor) { + return InvalidArgument("Unsupported type for Floor"); + } + + Status HandleFloor(HloInstruction* floor, HloInstruction* operand) override { + return HandleFloor(floor); + } Status HandleLog(HloInstruction* log, HloInstruction* operand) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[log], @@ -253,15 +333,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return std::log(elem_operand); })); return Status::OK(); - }; + } - Status HandleNot(HloInstruction* not_, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleNot(HloInstruction* not_) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[not_], ElementWiseUnaryOp(not_, [](ReturnT elem_operand) { return !elem_operand; })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleNot(HloInstruction* not_) { + return InvalidArgument("Unsupported type for Not"); + } + + Status HandleNot(HloInstruction* not_, HloInstruction* operand) override { + return HandleNot(not_); + } Status HandleNegate(HloInstruction* negate, HloInstruction* operand) override { @@ -270,16 +364,36 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return -elem_operand; })); return Status::OK(); - }; + } - Status HandleSign(HloInstruction* sign, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleSign(HloInstruction* sign) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign], ElementWiseUnaryOp(sign, [](ReturnT elem_operand) { return (ReturnT(0) < elem_operand) - (elem_operand < ReturnT(0)); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleSign(HloInstruction* sign) { + TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign], + ElementWiseUnaryOp(sign, [](ReturnT elem_operand) { + auto abs_val = std::abs(elem_operand); + return 0 == abs_val ? ReturnT(0) + : elem_operand / abs_val; + })); + return Status::OK(); + } + + Status HandleSign(HloInstruction* sign, HloInstruction* operand) override { + return HandleSign(sign); + } Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[tanh], @@ -287,7 +401,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return std::tanh(elem_operand); })); return Status::OK(); - }; + } Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, HloInstruction* rhs) override { @@ -297,7 +411,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return lhs_elem * rhs_elem; })); return Status::OK(); - }; + } Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs, HloInstruction* rhs) override { @@ -307,7 +421,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return lhs_elem - rhs_elem; })); return Status::OK(); - }; + } Status HandleAdd(HloInstruction* add, HloInstruction* lhs, HloInstruction* rhs) override { @@ -317,7 +431,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return lhs_elem + rhs_elem; })); return Status::OK(); - }; + } Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, HloInstruction* rhs) override { @@ -327,25 +441,53 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return lhs_elem / rhs_elem; })); return Status::OK(); - }; + } - Status HandleMaximum(HloInstruction* maximum) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMaximum(HloInstruction* maximum) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[maximum], ElementWiseBinaryOp(maximum, [](ReturnT lhs, ReturnT rhs) { return std::fmax(lhs, rhs); })); return Status::OK(); - }; + } - Status HandleMinimum(HloInstruction* minimum) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMaximum(HloInstruction* maximum) { + return InvalidArgument("Unsupported type for Maximum"); + } + + Status HandleMaximum(HloInstruction* maximum) override { + return HandleMaximum(maximum); + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMinimum(HloInstruction* minimum) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[minimum], ElementWiseBinaryOp(minimum, [](ReturnT lhs_el, ReturnT rhs_el) { return std::fmin(lhs_el, rhs_el); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMinimum(HloInstruction* minimum) { + return InvalidArgument("Unsupported type for Minimum"); + } + + Status HandleMinimum(HloInstruction* minimum) override { + return HandleMinimum(minimum); + } Status HandlePower(HloInstruction* power, HloInstruction* lhs, HloInstruction* rhs) override { @@ -355,37 +497,79 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return std::pow(lhs_el, rhs_el); })); return Status::OK(); - }; + } - Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, - HloInstruction* rhs) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRemainder(HloInstruction* remainder) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[remainder], ElementWiseBinaryOp(remainder, [](ReturnT lhs_el, ReturnT rhs_el) { return std::fmod(lhs_el, rhs_el); })); return Status::OK(); - }; + } - Status HandleAnd(HloInstruction* and_, HloInstruction* lhs, - HloInstruction* rhs) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRemainder(HloInstruction* remainder) { + return InvalidArgument("Unsupported type for Remainder"); + } + + Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, + HloInstruction* rhs) override { + return HandleRemainder(remainder); + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleAnd(HloInstruction* and_) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[and_], ElementWiseBinaryOp(and_, [](ReturnT lhs_el, ReturnT rhs_el) { return lhs_el && rhs_el; })); return Status::OK(); - }; + } - Status HandleOr(HloInstruction* or_, HloInstruction* lhs, - HloInstruction* rhs) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleAnd(HloInstruction* and_) { + return InvalidArgument("Unsupported type for And"); + } + + Status HandleAnd(HloInstruction* and_, HloInstruction* lhs, + HloInstruction* rhs) override { + return HandleAnd(and_); + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleOr(HloInstruction* or_) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[or_], ElementWiseBinaryOp(or_, [](ReturnT lhs_el, ReturnT rhs_el) { return lhs_el || rhs_el; })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleOr(HloInstruction* or_) { + return InvalidArgument("Unsupported type for Or"); + } + + Status HandleOr(HloInstruction* or_, HloInstruction* lhs, + HloInstruction* rhs) override { + return HandleOr(or_); + } template (shrl, lhs, rhs); } + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override { + HloInstruction* arg, HloInstruction* max) { std::function clamp_op = [](ReturnT low, ReturnT high, ReturnT value) { return std::fmax(low, std::fmin(value, high)); @@ -483,7 +670,20 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { TF_ASSIGN_OR_RETURN(parent_->evaluated_[clamp], ElementWiseTernaryOp(clamp, std::move(clamp_op))); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleClamp(HloInstruction* clamp, HloInstruction* min, + HloInstruction* arg, HloInstruction* max) { + return InvalidArgument("Unsupported type for Clamp"); + } + + Status HandleClamp(HloInstruction* clamp, HloInstruction* min, + HloInstruction* arg, HloInstruction* max) override { + return HandleClamp(clamp, min, arg, max); + } Status HandleSelect(HloInstruction* select, HloInstruction* pred, HloInstruction* on_true, @@ -499,7 +699,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { TF_ASSIGN_OR_RETURN(parent_->evaluated_[select], ElementWiseTernaryOp(select, std::move(select_op))); return Status::OK(); - }; + } Status HandleReverse(HloInstruction* reverse, HloInstruction* operand) override { @@ -529,7 +729,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[reverse] = std::move(result); return Status::OK(); - }; + } Status HandleConvolution(HloInstruction* conv, HloInstruction* lhs, HloInstruction* rhs, const Window& window) override { @@ -652,7 +852,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[conv] = std::move(result); return Status::OK(); - }; + } Status HandleDot(HloInstruction* dot, HloInstruction* lhs, HloInstruction* rhs) override { @@ -719,7 +919,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[dot] = std::move(result); return Status::OK(); - }; + } Status HandlePad(HloInstruction* pad) override { CHECK(!ShapeUtil::IsTuple(pad->operand(0)->shape())); @@ -788,7 +988,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[pad] = std::move(result); return Status::OK(); - }; + } Status HandleDynamicSlice(HloInstruction* dynamic_slice, HloInstruction* operand, @@ -841,7 +1041,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } return Status::OK(); - }; + } Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, HloInstruction* operand, @@ -897,7 +1097,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } return Status::OK(); - }; + } Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, HloInstruction* init_value, @@ -985,7 +1185,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[reduce] = std::move(result); return Status::OK(); - }; + } Status HandleReduceWindow(HloInstruction* reduce_window, HloInstruction* operand, const Window& window, @@ -1072,7 +1272,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[reduce_window] = std::move(result); return Status::OK(); - }; + } Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override { const Shape& shape = slice->shape(); @@ -1101,7 +1301,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { TF_RETURN_IF_ERROR(result->Populate(func)); parent_->evaluated_[slice] = std::move(result); return Status::OK(); - }; + } private: template @@ -1244,35 +1444,33 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } HloEvaluator* parent_; -}; // namespace xla +}; // class HloEvaluator::TypedVisitor HloEvaluator::HloEvaluator() { typed_visitors_[PRED] = MakeUnique>(this); typed_visitors_[U8] = MakeUnique>(this); typed_visitors_[U16] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: U16."); + return Unimplemented("HloEvaluator: unhandled primitive type: U16."); }); typed_visitors_[U32] = MakeUnique>(this); typed_visitors_[U64] = MakeUnique>(this); typed_visitors_[S8] = MakeUnique>(this); typed_visitors_[S16] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: S16."); + return Unimplemented("HloEvaluator: unhandled primitive type: S16."); }); typed_visitors_[S32] = MakeUnique>(this); typed_visitors_[S64] = MakeUnique>(this); typed_visitors_[F16] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: F16."); + return Unimplemented("HloEvaluator: unhandled primitive type: F16."); }); typed_visitors_[F32] = MakeUnique>(this); typed_visitors_[F64] = MakeUnique>(this); - typed_visitors_[C64] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: C64."); - }); + typed_visitors_[C64] = MakeUnique>(this); typed_visitors_[TUPLE] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: TUPLE."); + return Unimplemented("HloEvaluator: unhandled primitive type: TUPLE."); }); typed_visitors_[OPAQUE] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: OPAQUE."); + return Unimplemented("HloEvaluator: unhandled primitive type: OPAQUE."); }); } @@ -1573,6 +1771,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode, evaluated_[compare], Compare(compare->shape(), opcode, lhs_literal, rhs_literal)); } break; + case C64: { + TF_ASSIGN_OR_RETURN(evaluated_[compare], + Compare(compare->shape(), opcode, + lhs_literal, rhs_literal)); + } break; default: LOG(FATAL) << "HandleCompare: unknown primitive type: " << PrimitiveType_Name(lhs->shape().element_type()); diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index 7b9cbeb6f4..d0202556bc 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -826,8 +826,10 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kAbs: case HloOpcode::kRoundNearestAfz: case HloOpcode::kAdd: + case HloOpcode::kAtan2: case HloOpcode::kCeil: case HloOpcode::kClamp: + case HloOpcode::kComplex: case HloOpcode::kConvert: case HloOpcode::kCos: case HloOpcode::kDivide: @@ -836,6 +838,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kFloor: case HloOpcode::kGe: case HloOpcode::kGt: + case HloOpcode::kImag: case HloOpcode::kIndex: case HloOpcode::kIsFinite: case HloOpcode::kLe: @@ -850,6 +853,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kNe: case HloOpcode::kNegate: case HloOpcode::kPower: + case HloOpcode::kReal: case HloOpcode::kRemainder: case HloOpcode::kShiftLeft: case HloOpcode::kShiftRightArithmetic: diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 272f573623..1a03e7ee92 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -219,10 +219,12 @@ HloInstruction::CreateGetTupleElement(const Shape& shape, case HloOpcode::kCos: case HloOpcode::kExp: case HloOpcode::kFloor: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kSort: @@ -241,26 +243,28 @@ HloInstruction::CreateGetTupleElement(const Shape& shape, // Only certain opcodes are supported with CreateBinary: opcodes of binary // instructions with no auxiliary fields. switch (opcode) { - case (HloOpcode::kAdd): - case (HloOpcode::kDivide): - case (HloOpcode::kDot): - case (HloOpcode::kEq): - case (HloOpcode::kGe): - case (HloOpcode::kGt): - case (HloOpcode::kLe): - case (HloOpcode::kLt): - case (HloOpcode::kMaximum): - case (HloOpcode::kMinimum): - case (HloOpcode::kMultiply): - case (HloOpcode::kNe): - case (HloOpcode::kPower): - case (HloOpcode::kRemainder): - case (HloOpcode::kSubtract): - case (HloOpcode::kAnd): - case (HloOpcode::kOr): - case (HloOpcode::kShiftLeft): - case (HloOpcode::kShiftRightArithmetic): - case (HloOpcode::kShiftRightLogical): + case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kDivide: + case HloOpcode::kComplex: + case HloOpcode::kDot: + case HloOpcode::kEq: + case HloOpcode::kGe: + case HloOpcode::kGt: + case HloOpcode::kLe: + case HloOpcode::kLt: + case HloOpcode::kMaximum: + case HloOpcode::kMinimum: + case HloOpcode::kMultiply: + case HloOpcode::kNe: + case HloOpcode::kPower: + case HloOpcode::kRemainder: + case HloOpcode::kSubtract: + case HloOpcode::kAnd: + case HloOpcode::kOr: + case HloOpcode::kShiftLeft: + case HloOpcode::kShiftRightArithmetic: + case HloOpcode::kShiftRightLogical: break; default: LOG(FATAL) << "Invalid binary instruction opcode " @@ -978,11 +982,13 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kCopy: case HloOpcode::kCos: case HloOpcode::kExp: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kFloor: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kSort: @@ -992,6 +998,8 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( break; // Binary ops. case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kMultiply: case HloOpcode::kSubtract: @@ -1403,10 +1411,12 @@ bool HloInstruction::IdenticalSlowPath( // The result of these instructions only depend upon their opcode and // operands. case HloOpcode::kAbs: + case HloOpcode::kAtan2: case HloOpcode::kRoundNearestAfz: case HloOpcode::kAdd: case HloOpcode::kCeil: case HloOpcode::kClamp: + case HloOpcode::kComplex: case HloOpcode::kCopy: case HloOpcode::kCos: case HloOpcode::kCrossReplicaSum: @@ -1417,6 +1427,7 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kFloor: case HloOpcode::kGe: case HloOpcode::kGt: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLe: case HloOpcode::kLog: @@ -1430,6 +1441,7 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kNe: case HloOpcode::kNegate: case HloOpcode::kPower: + case HloOpcode::kReal: case HloOpcode::kRemainder: case HloOpcode::kSelect: case HloOpcode::kShiftLeft: @@ -2117,6 +2129,8 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { switch (opcode_) { case HloOpcode::kAbs: return visitor->HandleAbs(this, operands_[0]); + case HloOpcode::kAtan2: + return visitor->HandleAtan2(this, operands_[0], operands_[1]); case HloOpcode::kRoundNearestAfz: return visitor->HandleRound(this); case HloOpcode::kBatchNormTraining: @@ -2140,6 +2154,8 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kLt: case HloOpcode::kNe: return visitor->HandleCompare(this, opcode_, operands_[0], operands_[1]); + case HloOpcode::kComplex: + return visitor->HandleComplex(this, operands_[0], operands_[1]); case HloOpcode::kAdd: return visitor->HandleAdd(this, operands_[0], operands_[1]); case HloOpcode::kDivide: @@ -2214,6 +2230,10 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { return visitor->HandleCos(this, operands_[0]); case HloOpcode::kSin: return visitor->HandleSin(this, operands_[0]); + case HloOpcode::kReal: + return visitor->HandleReal(this, operands_[0]); + case HloOpcode::kImag: + return visitor->HandleImag(this, operands_[0]); case HloOpcode::kIsFinite: return visitor->HandleIsFinite(this, operands_[0]); case HloOpcode::kNot: @@ -2305,7 +2325,7 @@ static Status PostOrderDFS(HloInstruction* root, DfsHloVisitor* visitor, // // We need to keep track of both the id and the instruction because // instructions can get deleted while they are on the stack, so we - // can't always use the (potentiall dead) instruction object to grab + // can't always use the (potentially dead) instruction object to grab // its id. DFSStack dfs_stack; dfs_stack.emplace_back(root->unique_id(), root); @@ -2505,6 +2525,7 @@ bool HloInstruction::IsElementwiseBinary() const { // Binary elementwise operations. If you update this, please update // IsElementwise() accordingly. case HloOpcode::kAdd: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kEq: case HloOpcode::kGe: @@ -2537,6 +2558,7 @@ bool HloInstruction::IsElementwise() const { // Unary elementwise operations. case HloOpcode::kAbs: + case HloOpcode::kAtan2: case HloOpcode::kRoundNearestAfz: case HloOpcode::kCeil: case HloOpcode::kConvert: @@ -2544,10 +2566,12 @@ bool HloInstruction::IsElementwise() const { case HloOpcode::kCos: case HloOpcode::kExp: case HloOpcode::kFloor: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kReducePrecision: case HloOpcode::kSign: case HloOpcode::kSin: @@ -2557,6 +2581,7 @@ bool HloInstruction::IsElementwise() const { // Binary elementwise operations, the same as in IsElementwiseBinary(). // If you update this, please update IsElementwiseBinary() accordingly. case HloOpcode::kAdd: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kEq: case HloOpcode::kGe: diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc index e9000a8462..2f2263f70d 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.cc +++ b/tensorflow/compiler/xla/service/hlo_opcode.cc @@ -33,6 +33,8 @@ string HloOpcodeString(HloOpcode opcode) { return "abs"; case HloOpcode::kAdd: return "add"; + case HloOpcode::kAtan2: + return "atan2"; case HloOpcode::kBatchNormTraining: return "batch-norm-training"; case HloOpcode::kBatchNormInference: @@ -47,6 +49,8 @@ string HloOpcodeString(HloOpcode opcode) { return "call"; case HloOpcode::kClamp: return "clamp"; + case HloOpcode::kComplex: + return "complex"; case HloOpcode::kConcatenate: return "concatenate"; case HloOpcode::kConstant: @@ -87,6 +91,8 @@ string HloOpcodeString(HloOpcode opcode) { return "get-tuple-element"; case HloOpcode::kGt: return "greater-than"; + case HloOpcode::kImag: + return "imag"; case HloOpcode::kIndex: return "index"; case HloOpcode::kInfeed: @@ -125,6 +131,8 @@ string HloOpcodeString(HloOpcode opcode) { return "parameter"; case HloOpcode::kPower: return "power"; + case HloOpcode::kReal: + return "real"; case HloOpcode::kRecv: return "recv"; case HloOpcode::kReduce: diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h index c603c57e62..8090e4c82e 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.h +++ b/tensorflow/compiler/xla/service/hlo_opcode.h @@ -31,6 +31,7 @@ namespace xla { enum class HloOpcode { kAbs, kAdd, + kAtan2, kBatchNormGrad, kBatchNormInference, kBatchNormTraining, @@ -39,6 +40,7 @@ enum class HloOpcode { kCall, kCeil, kClamp, + kComplex, kConcatenate, kConstant, kConvert, @@ -58,6 +60,7 @@ enum class HloOpcode { kGe, kGetTupleElement, kGt, + kImag, kIndex, kInfeed, kIsFinite, @@ -77,6 +80,7 @@ enum class HloOpcode { kPad, kParameter, kPower, + kReal, kRecv, kReduce, kReducePrecision, diff --git a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc index ed7b6c71bc..53bd46a641 100644 --- a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc +++ b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc @@ -59,6 +59,7 @@ StatusOr HloPassPipeline::Run(HloModule* module) { for (auto& invariant_checker : invariant_checkers_) { VLOG(1) << " Invariant checker " << invariant_checker->name(); StatusOr changed_status = invariant_checker->Run(module); + VLOG(1) << " Invariant checker done " << invariant_checker->name(); if (!changed_status.ok()) { VLOG(2) << "Module failed invariant check:"; XLA_VLOG_LINES(2, module->ToString()); diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 35dff4a957..f3a098057b 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -64,6 +64,10 @@ class ShapeVerifier : public DfsHloVisitor { } Status HandleConvert(HloInstruction* convert) override { + if (ShapeUtil::ElementIsComplex(convert->operand(0)->shape())) { + TF_RET_CHECK(ShapeUtil::ElementIsComplex(convert->shape())) + << "Unsupported complex->real kConvert"; + } return CheckShape(convert, ShapeInference::InferConvertShape( convert->operand(0)->shape(), convert->shape().element_type())); diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc index 0271f41697..fae3ca8ad2 100644 --- a/tensorflow/compiler/xla/service/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/instruction_fusion.cc @@ -32,17 +32,16 @@ namespace xla { const HloInstruction& instruction) { switch (instruction.opcode()) { // Cheap instructions. - case HloOpcode::kAbs: case HloOpcode::kAdd: case HloOpcode::kBitcast: case HloOpcode::kBroadcast: case HloOpcode::kCeil: case HloOpcode::kClamp: + case HloOpcode::kComplex: case HloOpcode::kConcatenate: case HloOpcode::kConstant: case HloOpcode::kConvert: case HloOpcode::kCopy: - case HloOpcode::kCos: case HloOpcode::kDynamicSlice: case HloOpcode::kDynamicUpdateSlice: case HloOpcode::kEq: @@ -50,6 +49,7 @@ namespace xla { case HloOpcode::kGe: case HloOpcode::kGetTupleElement: case HloOpcode::kGt: + case HloOpcode::kImag: case HloOpcode::kInfeed: case HloOpcode::kIsFinite: case HloOpcode::kLe: @@ -64,6 +64,7 @@ namespace xla { case HloOpcode::kNegate: case HloOpcode::kOutfeed: case HloOpcode::kPad: + case HloOpcode::kReal: case HloOpcode::kReducePrecision: case HloOpcode::kReshape: case HloOpcode::kReverse: @@ -72,15 +73,21 @@ namespace xla { case HloOpcode::kShiftLeft: case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: - case HloOpcode::kSign: - case HloOpcode::kSin: case HloOpcode::kSlice: case HloOpcode::kSubtract: case HloOpcode::kTranspose: case HloOpcode::kTuple: return false; + // Cheap instructions for reals, but expensive for complex. + case HloOpcode::kAbs: + case HloOpcode::kCos: + case HloOpcode::kSign: + case HloOpcode::kSin: + return ShapeUtil::ElementIsComplex(instruction.shape()); + // Expensive instructions. + case HloOpcode::kAtan2: case HloOpcode::kBatchNormTraining: case HloOpcode::kBatchNormInference: case HloOpcode::kBatchNormGrad: diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc index d286c49d68..a2af2580ff 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc @@ -75,7 +75,7 @@ Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) { Status FusedIrEmitter::HandleConstant(HloInstruction* constant, const Literal& literal) { llvm::Constant* initializer = - llvm_ir::ConvertLiteralToIrConstant(literal, ir_builder_); + llvm_ir::ConvertLiteralToIrConstant(literal, module_); llvm::GlobalVariable* global = new llvm::GlobalVariable( *ir_builder_->GetInsertBlock()->getModule(), initializer->getType(), /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, initializer, @@ -101,7 +101,7 @@ Status FusedIrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, // Emit code to lookup tuple element pointer, and store it in 'gte_values_'. llvm::Value* tuple_element_ptr = llvm_ir::EmitGetTupleElement( get_tuple_element->shape(), get_tuple_element->tuple_index(), - /*alignment=*/1, it->second, ir_builder_); + /*alignment=*/1, it->second, ir_builder_, module_); gte_values_.insert(std::make_pair(get_tuple_element, tuple_element_ptr)); // Emit code to read base tuple element array (if non-tuple shaped). if (!ShapeUtil::IsTuple(get_tuple_element->shape())) { @@ -134,7 +134,7 @@ Status FusedIrEmitter::HandleTuple( std::vector operand_elemental_ir_types; for (HloInstruction* operand : operands) { operand_elemental_ir_types.push_back(llvm_ir::PrimitiveTypeToIrType( - operand->shape().element_type(), ir_builder_)); + operand->shape().element_type(), module_)); } generators_[tuple] = [=](const IrArray::Index& index) -> StatusOr { diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h index a24e104067..a44da51378 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h +++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h @@ -42,7 +42,8 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault { ElementalIrEmitter* elemental_emitter) : parameter_arrays_(parameter_arrays), elemental_emitter_(elemental_emitter), - ir_builder_(elemental_emitter->ir_builder()) {} + ir_builder_(elemental_emitter->ir_builder()), + module_(elemental_emitter->module()) {} Status DefaultAction(HloInstruction* hlo) override; @@ -85,6 +86,7 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault { // Borrowed llvm::IRBuilder<>* ir_builder_; + llvm::Module* module_; // Map from instruction pointers to functions to generate elements of their // outputs diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc index 6a00a565c6..e3f98ac13e 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc @@ -229,9 +229,11 @@ llvm::Value* IrArray::EmitArrayElementAddress( } if (!is_implicit_broadcast && index.LinearValidOnShape(*shape_)) { + llvm::Module* module = + ir_builder->GetInsertBlock()->getParent()->getParent(); return ir_builder->CreateInBoundsGEP( ir_builder->CreateBitCast( - base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), ir_builder) + base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), module) ->getPointerTo()), {index.linear()}, llvm_ir::AsStringRef(name)); } @@ -281,7 +283,8 @@ void IrArray::EmitWriteArrayElement(const Index& index, llvm::Value* value, IrArray IrArray::CastToShape(const Shape& new_shape, llvm::IRBuilder<>* ir_builder) const { - llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, ir_builder); + llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent(); + llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, module); return IrArray( ir_builder->CreatePointerCast(base_ptr_, new_ir_type->getPointerTo()), new_shape); diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index 8e188e7ae8..5dff4b5778 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Operator.h" #include "llvm/Target/TargetOptions.h" @@ -38,6 +39,19 @@ limitations under the License. namespace xla { namespace llvm_ir { +namespace { + +// Note, this function is only useful in an insertion context; in a global +// (e.g. constants) context it will CHECK fail. +llvm::Module* ModuleFromIRBuilder(llvm::IRBuilder<>* ir_builder) { + auto block = CHECK_NOTNULL(ir_builder->GetInsertBlock()); + auto fn = CHECK_NOTNULL(block->getParent()); + auto module = CHECK_NOTNULL(fn->getParent()); + return module; +} + +} // namespace + string AsString(const std::string& str) { return string(str.data(), str.length()); } @@ -63,7 +77,7 @@ llvm::Value* EmitCallToIntrinsic( for (auto type : overloaded_types) { types.push_back(type); } - llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent(); + llvm::Module* module = ModuleFromIRBuilder(ir_builder); llvm::Function* intrinsic = llvm::Intrinsic::getDeclaration(module, intrinsic_id, types); std::vector operands_vec; @@ -119,38 +133,53 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index, } llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type, - llvm::IRBuilder<>* ir_builder) { + llvm::Module* module) { switch (element_type) { case PRED: case S8: case U8: - return ir_builder->getInt8Ty(); + return llvm::Type::getInt8Ty(module->getContext()); case S16: case U16: - return ir_builder->getInt16Ty(); + return llvm::Type::getInt16Ty(module->getContext()); case S32: case U32: - return ir_builder->getInt32Ty(); + return llvm::Type::getInt32Ty(module->getContext()); case S64: case U64: - return ir_builder->getInt64Ty(); + return llvm::Type::getInt64Ty(module->getContext()); case F32: - return ir_builder->getFloatTy(); + return llvm::Type::getFloatTy(module->getContext()); case F64: - return ir_builder->getDoubleTy(); + return llvm::Type::getDoubleTy(module->getContext()); + case C64: { + auto cplx_t = module->getTypeByName("complex64"); + if (cplx_t == nullptr) { + // C++ standard dictates the memory layout of std::complex is contiguous + // real followed by imaginary. C++11 section 26.4 [complex.numbers]: + // If z is an lvalue expression of type cv std::complex then the + // expression reinterpret_cast(z) shall be well-formed, + // reinterpret_cast(z)[0] shall designate the real part of + // z, and reinterpret_cast(z)[1] shall designate the + // imaginary part of z. + return llvm::StructType::create( + "complex64", llvm::Type::getFloatTy(module->getContext()), + llvm::Type::getFloatTy(module->getContext())); + } + return cplx_t; + } // A Tuple contains an array of pointers. Use i8*. case TUPLE: // An Opaque is like a void*, use i8*. case OPAQUE: - return ir_builder->getInt8PtrTy(); + return llvm::Type::getInt8PtrTy(module->getContext()); default: LOG(FATAL) << "unsupported type " << element_type; } } -llvm::Type* ShapeToIrType(const Shape& shape, llvm::IRBuilder<>* ir_builder) { - llvm::Type* result_type = - PrimitiveTypeToIrType(shape.element_type(), ir_builder); +llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module) { + llvm::Type* result_type = PrimitiveTypeToIrType(shape.element_type(), module); if (ShapeUtil::IsTuple(shape)) { // A tuple buffer is an array of pointers. result_type = llvm::ArrayType::get(result_type, shape.tuple_shapes_size()); @@ -197,10 +226,10 @@ namespace { // value down to zero). llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, std::vector* multi_index, - llvm::IRBuilder<>* ir_builder) { + llvm::Module* module) { const Shape& shape = literal.shape(); llvm::Type* ir_element_type = - llvm_ir::PrimitiveTypeToIrType(shape.element_type(), ir_builder); + llvm_ir::PrimitiveTypeToIrType(shape.element_type(), module); if (dimension_index == -1) { // Base case of the recursion. Index into the data field of the protobuf // with the multi index. @@ -238,6 +267,16 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, value = llvm::ConstantFP::get(ir_element_type, literal.Get(*multi_index)); break; + case C64: { + complex64 x = literal.Get(*multi_index); + value = llvm::ConstantStruct::get( + static_cast(ir_element_type), + llvm::ConstantFP::get(llvm_ir::PrimitiveTypeToIrType(F32, module), + x.real()), + llvm::ConstantFP::get(llvm_ir::PrimitiveTypeToIrType(F32, module), + x.imag())); + break; + } default: LOG(FATAL) << "unsupported type " << shape.element_type(); } @@ -256,8 +295,8 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, std::vector elements; for (int64 i = 0; i < shape.dimensions(dimension); ++i) { (*multi_index)[dimension] = i; - elements.push_back(LiteralToConstant(literal, dimension_index - 1, - multi_index, ir_builder)); + elements.push_back( + LiteralToConstant(literal, dimension_index - 1, multi_index, module)); } llvm::Type* element_type; @@ -279,11 +318,11 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, } // namespace llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal, - llvm::IRBuilder<>* ir_builder) { + llvm::Module* module) { std::vector multi_index(ShapeUtil::Rank(literal.shape()), 0); llvm::Constant* value = LiteralToConstant( literal, /*dimension_index=*/ShapeUtil::Rank(literal.shape()) - 1, - &multi_index, ir_builder); + &multi_index, module); return value; } @@ -380,7 +419,8 @@ llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate, // comparison_result is i1, but the NVPTX codegen incorrectly lowers i1 // arrays. So we extend it to i8 so that it's addressable. return ir_builder->CreateZExt( - comparison_result, llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder)); + comparison_result, + llvm_ir::PrimitiveTypeToIrType(PRED, ModuleFromIRBuilder(ir_builder))); } // Internal helper that is called from emitted code to log an int64 value with a diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h index 7a7d14da1e..304192b58e 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h @@ -127,11 +127,11 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index, // Returns the LLVM type which represents the given XLA primitive type. llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type, - llvm::IRBuilder<>* ir_builder); + llvm::Module* module); // Returns the LLVM type which represents the given XLA shape. For example, // if "shape" is [5 x [10 x f32]], the function returns [5 x [10 x float]]. -llvm::Type* ShapeToIrType(const Shape& shape, llvm::IRBuilder<>* ir_builder); +llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module); // Returns a value that represents a pointer to a global string constant that // encodes the shape as a serialized protobuf. @@ -149,7 +149,7 @@ StatusOr DecodeSelfDescribingShapeConstant(const void* shape_ptr, // Converts a given literal to an IR Constant. Literals have known constant // values at IR emission time. llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal, - llvm::IRBuilder<>* ir_builder); + llvm::Module* module); // Inserts an allocate of the requested type at the entry point of the // function that the builder is currently building. The insert point diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc index 6051cbfc6f..3a21eda357 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc @@ -31,14 +31,15 @@ namespace xla { namespace llvm_ir { void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true, - llvm::Value* on_false, llvm::IRBuilder<>* ir_builder) { + llvm::Value* on_false, llvm::IRBuilder<>* ir_builder, + llvm::Module* module) { CHECK(ShapeUtil::IsScalar(pred.GetShape())); llvm::LoadInst* pred_value = ir_builder->CreateLoad(pred.GetBasePointer(), "load_predicate_value"); llvm::Value* pred_cond = ir_builder->CreateICmpNE( pred_value, - llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, ir_builder), 0), + llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, module), 0), "boolean_predicate"); VLOG(2) << "HandleSelect for tuple:"; @@ -71,11 +72,11 @@ void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true, void EmitTuple(IrArray tuple, tensorflow::gtl::ArraySlice operands, - llvm::IRBuilder<>* ir_builder) { + llvm::IRBuilder<>* ir_builder, llvm::Module* module) { for (size_t i = 0; i < operands.size(); ++i) { auto* store = ir_builder->CreateStore( ir_builder->CreatePointerCast(operands[i], - PrimitiveTypeToIrType(TUPLE, ir_builder)), + PrimitiveTypeToIrType(TUPLE, module)), ir_builder->CreateInBoundsGEP( tuple.GetBasePointer(), {ir_builder->getInt64(0), ir_builder->getInt64(i)})); @@ -85,7 +86,8 @@ void EmitTuple(IrArray tuple, llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index, int alignment, llvm::Value* operand, - llvm::IRBuilder<>* ir_builder) { + llvm::IRBuilder<>* ir_builder, + llvm::Module* module) { llvm::Value* element_ptr = ir_builder->CreateInBoundsGEP( operand, {ir_builder->getInt64(0), ir_builder->getInt64(index)}); llvm::LoadInst* src_buffer = ir_builder->CreateLoad(element_ptr); @@ -98,7 +100,7 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index, } SetAlignmentMetadataForLoad(src_buffer, alignment); - llvm::Type* element_type = ShapeToIrType(target_shape, ir_builder); + llvm::Type* element_type = ShapeToIrType(target_shape, module); llvm::Value* ret_val = ir_builder->CreateBitCast(src_buffer, element_type->getPointerTo()); return ret_val; diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h index a75cdc8158..dbf9a14006 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h +++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h @@ -60,13 +60,14 @@ namespace llvm_ir { // tuple_on_true or tuple_on_false: // output[i] = pred ? tuple_on_true[i] : tuple_on_false[i] void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true, - llvm::Value* on_false, llvm::IRBuilder<>* ir_builder); + llvm::Value* on_false, llvm::IRBuilder<>* ir_builder, + llvm::Module* module); // A tuple is an array of pointers, one for each operand. Each pointer points to // the output buffer of its corresponding operand. void EmitTuple(IrArray tuple, tensorflow::gtl::ArraySlice operands, - llvm::IRBuilder<>* ir_builder); + llvm::IRBuilder<>* ir_builder, llvm::Module* module); // A tuple is an array of pointers, one for each operand. Each pointer points to // the output buffer of its corresponding operand. A GetTupleElement instruction @@ -74,7 +75,8 @@ void EmitTuple(IrArray tuple, // Returns an llvm value representing a pointer to the tuple element buffer. llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index, int alignment, llvm::Value* operand, - llvm::IRBuilder<>* ir_builder); + llvm::IRBuilder<>* ir_builder, + llvm::Module* module); } // namespace llvm_ir } // namespace xla diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index e41b7607c5..0458932a73 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -53,6 +53,8 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) { return UNOP_EXP; case HloOpcode::kFloor: return UNOP_FLOOR; + case HloOpcode::kImag: + return UNOP_IMAG; case HloOpcode::kIsFinite: return UNOP_IS_FINITE; case HloOpcode::kLog: @@ -61,6 +63,8 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) { return UNOP_NOT; case HloOpcode::kNegate: return UNOP_NEGATE; + case HloOpcode::kReal: + return UNOP_REAL; case HloOpcode::kRoundNearestAfz: return UNOP_ROUND_NEAREST_AFZ; case HloOpcode::kSign: @@ -81,6 +85,10 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) { // opcode. BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) { switch (opcode) { + case HloOpcode::kAtan2: + return BINOP_ATAN2; + case HloOpcode::kComplex: + return BINOP_COMPLEX; case HloOpcode::kDot: return BINOP_DOT; case HloOpcode::kMultiply: @@ -307,19 +315,41 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, switch (operation) { case UNOP_FLOOR: case UNOP_CEIL: + if (!ShapeUtil::ElementIsFloating(arg)) { + return InvalidArgument( + "expected element type in shape to be floating for floor/ceil " + "operation; got %s", + PrimitiveType_Name(arg.element_type()).c_str()); + } + return arg; case UNOP_COS: case UNOP_SIN: case UNOP_EXP: case UNOP_LOG: case UNOP_TANH: - if (!ShapeUtil::ElementIsFloating(arg)) { + if (!ShapeUtil::ElementIsFloating(arg) && + !ShapeUtil::ElementIsComplex(arg)) { return InvalidArgument( - "expected element type in shape to be floating for exp/log/tanh " - "operation; got %s", + "expected element type in shape to be floating or complex for " + "sin/cos/exp/log/tanh operation; got %s", PrimitiveType_Name(arg.element_type()).c_str()); } return arg; + case UNOP_REAL: + case UNOP_IMAG: + if (!ShapeUtil::ElementIsComplex(arg)) { + return InvalidArgument( + "expected element type in shape to be complex for real/imag " + "operation; got %s", + PrimitiveType_Name(arg.element_type()).c_str()); + } + return ShapeUtil::ChangeElementType(arg, F32); case UNOP_ABS: + if (ShapeUtil::ElementIsComplex(arg)) { + return ShapeUtil::ChangeElementType( + arg, primitive_util::ComplexComponentType(arg.element_type())); + } + return arg; case UNOP_NEGATE: case UNOP_ROUND_NEAREST_AFZ: case UNOP_SIGN: @@ -751,6 +781,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( case BINOP_MIN: case BINOP_SUB: case BINOP_ADD: + case BINOP_ATAN2: case BINOP_POW: case BINOP_DIV: case BINOP_REM: @@ -761,6 +792,22 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( return InferElementwiseBinaryOpShape(operation, lhs, rhs, broadcast_dimensions); + case BINOP_COMPLEX: { + if (!ShapeUtil::ElementIsFloating(lhs)) { + return InvalidArgument( + "expected element type in shape to be floating for complex compose " + "operation; got %s", + PrimitiveType_Name(lhs.element_type()).c_str()); + } + TF_ASSIGN_OR_RETURN(const Shape& shape, + InferElementwiseBinaryOpShape(operation, lhs, rhs, + broadcast_dimensions)); + if (lhs.element_type() == F32) { + return ShapeUtil::ChangeElementType(shape, C64); + } else { + return Unimplemented("complex component type not supported"); + } + } case BINOP_AND: case BINOP_OR: if (lhs.element_type() != PRED && diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc index 8df4a73229..d12f7bd145 100644 --- a/tensorflow/compiler/xla/service/shape_inference_test.cc +++ b/tensorflow/compiler/xla/service/shape_inference_test.cc @@ -35,6 +35,7 @@ class ShapeInferenceTest : public ::testing::Test { // Some handy scalar shapes. const Shape s32_ = ShapeUtil::MakeShape(S32, {}); const Shape f32_ = ShapeUtil::MakeShape(F32, {}); + const Shape f64_ = ShapeUtil::MakeShape(F64, {}); const Shape pred_ = ShapeUtil::MakeShape(PRED, {}); // Some handy vector and matrix shapes of F32 type. @@ -251,6 +252,44 @@ TEST_F(ShapeInferenceTest, ClampBadShapes) { .ok()); } +TEST_F(ShapeInferenceTest, Complex) { + auto complex_shape = [&](const Shape& lhs, const Shape& rhs, + const tensorflow::gtl::ArraySlice& bcast) { + return ShapeInference::InferBinaryOpShape(BinaryOperation::BINOP_COMPLEX, + lhs, rhs, bcast); + }; + // Inputs must be FP. + ASSERT_FALSE(complex_shape(s32_, s32_, {}).ok()); + ASSERT_FALSE(complex_shape(pred_, pred_, {}).ok()); + // Component types must match. + ASSERT_FALSE(complex_shape(f32_, f64_, {}).ok()); + // Only F32->C64 supported. + ASSERT_FALSE(complex_shape(f64_, f64_, {}).ok()); + // Validate correct uses. + Shape c64_32 = ShapeUtil::MakeShape(C64, {32}); + TF_ASSERT_OK_AND_ASSIGN(Shape result, complex_shape(f32_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, ShapeUtil::MakeShape(C64, {}))); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(vector_32_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32)); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(f32_, vector_32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32)); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(vector_32_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32)); + + Shape c64_32_64 = ShapeUtil::MakeShape(C64, {32, 64}); + TF_ASSERT_OK_AND_ASSIGN(result, + complex_shape(vector_64_, matrix_32_64_, {1})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); + TF_ASSERT_OK_AND_ASSIGN(result, + complex_shape(matrix_32_64_, vector_64_, {1})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); + TF_ASSERT_OK_AND_ASSIGN(result, + complex_shape(matrix_32_64_, matrix_32_64_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(matrix_32_64_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); +} + TEST_F(ShapeInferenceTest, VariadicOpTuplify) { StatusOr result = ShapeInference::InferVariadicOpShape( VariadicOperation::VAROP_TUPLE, {&s32_, &f32_}); diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index d818830f98..adf7972e0d 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -55,6 +55,8 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) { return HloOpcode::kExp; case UNOP_FLOOR: return HloOpcode::kFloor; + case UNOP_IMAG: + return HloOpcode::kImag; case UNOP_IS_FINITE: return HloOpcode::kIsFinite; case UNOP_LOG: @@ -63,6 +65,8 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) { return HloOpcode::kNot; case UNOP_NEGATE: return HloOpcode::kNegate; + case UNOP_REAL: + return HloOpcode::kReal; case UNOP_ROUND_NEAREST_AFZ: return HloOpcode::kRoundNearestAfz; case UNOP_SIGN: @@ -80,6 +84,10 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) { HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) { switch (binop) { + case BINOP_ATAN2: + return HloOpcode::kAtan2; + case BINOP_COMPLEX: + return HloOpcode::kComplex; case BINOP_DOT: return HloOpcode::kDot; case BINOP_MUL: diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index fa4f71414d..b5eb81dfc6 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -272,6 +272,7 @@ StatusOr MakeShapeWithLayoutInternal( case U16: case U32: case U64: + case C64: case TUPLE: case OPAQUE: return false; diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h index 7fe1445b94..7cfc276ec1 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.h +++ b/tensorflow/compiler/xla/tests/client_library_test_base.h @@ -361,8 +361,9 @@ void ClientLibraryTestBase::ComputeAndCompareR2( ComputationBuilder* builder, const Array2D& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error) { static_assert(std::is_same::value || - std::is_same::value, - "Floating point type required when specifying an ErrorSpec"); + std::is_same::value || + std::is_same::value, + "Float or complex type required when specifying an ErrorSpec"); std::unique_ptr expected_literal = Literal::CreateR2FromArray2D(expected); ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal, @@ -384,8 +385,9 @@ void ClientLibraryTestBase::ComputeAndCompareR3( ComputationBuilder* builder, const Array3D& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error) { static_assert(std::is_same::value || - std::is_same::value, - "Floating point type required when specifying an ErrorSpec"); + std::is_same::value || + std::is_same::value, + "Float or complex type required when specifying an ErrorSpec"); std::unique_ptr expected_literal = Literal::CreateR3FromArray3D(expected); ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal, @@ -407,8 +409,9 @@ void ClientLibraryTestBase::ComputeAndCompareR4( ComputationBuilder* builder, const Array4D& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error) { static_assert(std::is_same::value || - std::is_same::value, - "Floating point type required when specifying an ErrorSpec"); + std::is_same::value || + std::is_same::value, + "Float or complex type required when specifying an ErrorSpec"); std::unique_ptr expected_literal = Literal::CreateR4FromArray4D(expected); ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal, diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc index 224aa57899..cf089d748d 100644 --- a/tensorflow/compiler/xla/tests/dot_operation_test.cc +++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc @@ -347,7 +347,7 @@ XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTF) { TestNonsquareMatrixDot(kLhsRowMajor, kRhsRowMajor); } -TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTT) { +XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTT) { constexpr bool kLhsRowMajor = true; constexpr bool kRhsRowMajor = true; TestNonsquareMatrixDot(kLhsRowMajor, kRhsRowMajor); @@ -357,7 +357,11 @@ XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF64) { TestNonsquareMatrixDot(); } -TEST_F(DotOperationTest, ConcurrentMatMul) { +XLA_TEST_F(DotOperationTest, NonsquareMatrixDotC64) { + TestNonsquareMatrixDot(); +} + +XLA_TEST_F(DotOperationTest, ConcurrentMatMul) { ComputationBuilder builder(client_, TestName()); auto matrix1 = builder.ConstantR2({{1.0, 2.0}, {3.0, 4.0}}); auto matrix2 = builder.ConstantR2({{5.0, 6.0}, {7.0, 8.0}}); diff --git a/tensorflow/compiler/xla/tests/unary_op_test.cc b/tensorflow/compiler/xla/tests/unary_op_test.cc index efae13a43a..fa4192e928 100644 --- a/tensorflow/compiler/xla/tests/unary_op_test.cc +++ b/tensorflow/compiler/xla/tests/unary_op_test.cc @@ -41,7 +41,11 @@ class UnaryOpTest : public ClientLibraryTestBase { auto arg = builder.ConstantR1({}); auto abs = builder.Abs(arg); - ComputeAndCompareR1(&builder, {}, {}); + if (primitive_util::NativeToPrimitiveType() == C64) { + ComputeAndCompareR1(&builder, {}, {}); + } else { + ComputeAndCompareR1(&builder, {}, {}); + } } template @@ -80,14 +84,58 @@ int UnaryOpTest::inf() { return 2147483647; } +template <> +void UnaryOpTest::AbsTestHelper() { + ComputationBuilder builder(client_, TestName()); + auto arg = builder.ConstantR1({{-2, 0}, + {0, 25}, + {0, 0}, + {-0.3f, 0.4f}, + {0, inf()}, + {-inf(), 0}}); + auto abs = builder.Abs(arg); + + std::unique_ptr expected = + Literal::CreateR1({2, 25, 0, 0.5, inf(), inf()}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); +} + +template <> +void UnaryOpTest::SignTestHelper() { + ComputationBuilder builder(client_, TestName()); + auto arg = builder.ConstantR1( + {{-2, 0}, {0, 25}, {0, 0}, {static_cast(-0.0), 0}, {-1, 1}}); + auto sign = builder.Sign(arg); + + std::unique_ptr expected = Literal::CreateR1( + {{-1, 0}, {0, 1}, {0, 0}, {0, 0}, {-std::sqrt(0.5f), std::sqrt(0.5f)}}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); +} + +template <> +void UnaryOpTest::SignAbsTestHelper() { + ComputationBuilder builder(client_, TestName()); + auto arg = + builder.ConstantR1({{-2, 0}, {0, 25}, {0, 0}, {-0.4, 0.3}}); + auto sign = builder.Sign(arg); + auto abs = builder.Abs(arg); + builder.Sub(builder.Mul(sign, builder.ConvertElementType(abs, C64)), arg); + + std::unique_ptr expected = + Literal::CreateR1({0, 0, 0, 0}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); +} + XLA_TEST_F(UnaryOpTest, AbsTestR1Size0) { AbsSize0TestHelper(); AbsSize0TestHelper(); + AbsSize0TestHelper(); } XLA_TEST_F(UnaryOpTest, AbsTestR1) { AbsTestHelper(); AbsTestHelper(); + AbsTestHelper(); } XLA_TEST_F(UnaryOpTest, AbsTestR0) { @@ -98,34 +146,44 @@ XLA_TEST_F(UnaryOpTest, AbsTestR0) { auto absf = builder.Abs(argf); auto argf0 = builder.ConstantR0(-0.0f); auto absf0 = builder.Abs(argf0); - builder.Add(absf0, builder.Add(absf, builder.ConvertElementType( - absi, PrimitiveType::F32))); + auto argc = builder.ConstantR0({-0.3f, 0.4f}); + auto absc = builder.Abs(argc); + builder.Add(builder.Add(absc, absf0), + builder.Add(absf, builder.ConvertElementType(absi, F32))); - ComputeAndCompareR0(&builder, 8.0f, {}); + ComputeAndCompareR0(&builder, 8.5f, {}); } XLA_TEST_F(UnaryOpTest, SignTestR0) { ComputationBuilder builder(client_, TestName()); auto argi = builder.ConstantR0(-5); - auto absi = builder.Sign(argi); + auto sgni = builder.Sign(argi); // -1 auto argf = builder.ConstantR0(-4.0f); - auto absf = builder.Sign(argf); + auto sgnf = builder.Sign(argf); // -1 auto argf0 = builder.ConstantR0(-0.0f); - auto absf0 = builder.Sign(argf0); - builder.Add(absf0, builder.Add(absf, builder.ConvertElementType( - absi, PrimitiveType::F32))); - - ComputeAndCompareR0(&builder, -2.0f, {}); + auto sgnf0 = builder.Sign(argf0); // 0 + auto argc = builder.ConstantR0({-.3, .4}); + auto sgnc = builder.Sign(argc); // (-.6, .8) + builder.Add(sgnc, builder.ConvertElementType( + builder.Add(builder.Add(sgnf0, sgnf), + builder.ConvertElementType(sgni, F32)), + C64)); + + std::unique_ptr expected = + Literal::CreateR0({-2.6f, 0.8f}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); } XLA_TEST_F(UnaryOpTest, SignTestR1) { SignTestHelper(); SignTestHelper(); + SignTestHelper(); } XLA_TEST_F(UnaryOpTest, SignAbsTestR1) { SignAbsTestHelper(); SignAbsTestHelper(); + SignAbsTestHelper(); } XLA_TEST_F(UnaryOpTest, UnsignedAbsTestR1) { diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc index f4af03cc2f..d91404d73a 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -235,11 +235,13 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, case HloOpcode::kCopy: case HloOpcode::kCos: case HloOpcode::kExp: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kFloor: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kSort: @@ -256,6 +258,8 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, case HloOpcode::kDivide: case HloOpcode::kMultiply: case HloOpcode::kSubtract: + case HloOpcode::kAtan2: + case HloOpcode::kComplex: case HloOpcode::kEq: case HloOpcode::kGe: case HloOpcode::kGt: diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h index 3d78466107..3b19ca321c 100644 --- a/tensorflow/compiler/xla/types.h +++ b/tensorflow/compiler/xla/types.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_TYPES_H_ #define TENSORFLOW_COMPILER_XLA_TYPES_H_ +#include + #include "third_party/eigen3/Eigen/Core" #include "tensorflow/core/platform/types.h" @@ -35,7 +37,7 @@ using ::tensorflow::uint16; using ::tensorflow::uint32; using ::tensorflow::uint64; -typedef std::complex complex64; +using complex64 = std::complex; using ::Eigen::half; diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 0efa3d0014..fe47f85c12 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -49,7 +49,7 @@ enum PrimitiveType { F64 = 12; // Complex values of fixed width. - C64 = 15; + C64 = 15; // Paired F32 (real, imag), as in std::complex. // A tuple is a polymorphic sequence; e.g. a shape that holds different // sub-shapes. They are used for things like returning multiple values from a @@ -667,6 +667,12 @@ enum UnaryOperation { // Elementwise, rounds x to nearest integral value, rounding half-way cases // away from zero. UNOP_ROUND_NEAREST_AFZ = 14; + + // Elementwise, extract real component of complex x. + UNOP_REAL = 15; + + // Elementwise, extract real component of complex x. + UNOP_IMAG = 16; } message UnaryOpRequest { @@ -721,6 +727,12 @@ enum BinaryOperation { BINOP_SHIFT_LEFT = 20; BINOP_SHIFT_RIGHT_ARITHMETIC = 21; BINOP_SHIFT_RIGHT_LOGICAL = 22; + + // Complex from real, imag. + BINOP_COMPLEX = 23; + + // Computes the 4-quadrant arctangent of the y, x input arguments. + BINOP_ATAN2 = 24; } message BinaryOpRequest { -- GitLab From c22973867f742bb1395a4cdb87deb8f7cb21d1a5 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Fri, 27 Oct 2017 09:32:39 -0700 Subject: [PATCH 458/573] Delete bad links (md links not supported in html blocks). PiperOrigin-RevId: 173680417 --- tensorflow/docs_src/get_started/linear_regression.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tensorflow/docs_src/get_started/linear_regression.md b/tensorflow/docs_src/get_started/linear_regression.md index 7cfff8db15..45cb9d829c 100644 --- a/tensorflow/docs_src/get_started/linear_regression.md +++ b/tensorflow/docs_src/get_started/linear_regression.md @@ -4,32 +4,28 @@ This unit provides the following short examples demonstrating how to implement regression in Estimators:

Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.3.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.3.0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.368
tensorflow-1.4.0rc0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.4.0rc0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.368
tensorflow-1.2.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.2.0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.35.18
tensorflow-1.1.0CPU3.5MSVC 2015 update 3Cmake v3.6.3N/AN/A
- + - - - - @@ -96,7 +92,7 @@ During training, all three programs output the following information: For example, here's some possible output for the `linear_regressor.py` program: -```bsh +``` None INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpAObiz9/model.ckpt. INFO:tensorflow:loss = 161.308, step = 1 INFO:tensorflow:global_step/sec: 1557.24 -- GitLab From 96dc501cd9e9aefd2766cc02f8d0e5436d198bb8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2017 14:17:32 +0000 Subject: [PATCH 459/573] Fix incorrect annotation tag in tf.Variable In tf.Variable the annotation tag of `@compatiblity` should be `@compatibility` --- tensorflow/python/ops/variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index fd0aee3c33..5d18979bef 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -119,7 +119,7 @@ class Variable(object): various `Optimizer` classes use this collection as the default list of variables to optimize. - @compatiblity(eager) + @compatibility(eager) `tf.Variable` is not compatible with eager execution. Use `tfe.Variable` instead which is compatable with both eager execution and graph construction. See [the TensorFlow Eager Execution -- GitLab From 7d7b2ec58e9625c5f2320c67516a5e0aff06d0d3 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2017 14:20:58 +0000 Subject: [PATCH 460/573] Also fixes `@end_compatiblity` -> `@end_compatibility` Signed-off-by: Yong Tang --- tensorflow/python/ops/variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 5d18979bef..e78139edc2 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -125,7 +125,7 @@ class Variable(object): and graph construction. See [the TensorFlow Eager Execution guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) for details on how variables work in eager execution. - @end_compatiblity + @end_compatibility """ def __init__(self, -- GitLab From 5120e75cffc1bef4766cc8d53f2e13a00750204a Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 27 Oct 2017 17:02:27 +0000 Subject: [PATCH 461/573] Move `@compatibility(eager)` from class docstring to __init__ docstring Signed-off-by: Yong Tang --- tensorflow/python/ops/variables.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index e78139edc2..57b27051af 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -118,14 +118,6 @@ class Variable(object): `trainable_variables()` returns the contents of this collection. The various `Optimizer` classes use this collection as the default list of variables to optimize. - - @compatibility(eager) - `tf.Variable` is not compatible with eager execution. Use - `tfe.Variable` instead which is compatable with both eager execution - and graph construction. See [the TensorFlow Eager Execution - guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) - for details on how variables work in eager execution. - @end_compatibility """ def __init__(self, @@ -197,6 +189,14 @@ class Variable(object): ValueError: If the initial value is not specified, or does not have a shape and `validate_shape` is `True`. RuntimeError: If eager execution is enabled. + + @compatibility(eager) + `tf.Variable` is not compatible with eager execution. Use + `tfe.Variable` instead which is compatable with both eager execution + and graph construction. See [the TensorFlow Eager Execution + guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) + for details on how variables work in eager execution. + @end_compatibility """ if not context.in_graph_mode(): raise RuntimeError("tf.Variable not supported in Eager mode. " -- GitLab From 7775a6604330497c81d8290037b7f59ffffafec0 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Fri, 27 Oct 2017 10:15:31 -0700 Subject: [PATCH 462/573] Internal Change PiperOrigin-RevId: 173685895 --- tensorflow/workspace.bzl | 9 +++++++++ .../arm_neon_2_x86_sse.BUILD | 0 2 files changed, 9 insertions(+) rename {tensorflow/opensource_only => third_party}/arm_neon_2_x86_sse.BUILD (100%) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b9651a92f7..c0eb87a744 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -723,6 +723,15 @@ def tf_workspace(path_prefix="", tf_repo_name=""): deps = ["@com_google_guava"], ) + java_import_external( + name = "javax_validation", + jar_sha256 = "e459f313ebc6db2483f8ceaad39af07086361b474fa92e40f442e8de5d9895dc", + jar_urls = [ + "http://repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar", + ], + licenses = ["notice"], # Apache 2.0 + ) + native.new_http_archive( name = "com_google_pprof", urls = [ diff --git a/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD b/third_party/arm_neon_2_x86_sse.BUILD similarity index 100% rename from tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD rename to third_party/arm_neon_2_x86_sse.BUILD -- GitLab From 6b05b36cd2c809a1fd581341be51076ab0d05e8e Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Fri, 27 Oct 2017 10:29:36 -0700 Subject: [PATCH 463/573] Generalizing sloppy_interleave, making sloppiness an option. PiperOrigin-RevId: 173687797 --- tensorflow/contrib/data/__init__.py | 2 +- .../contrib/data/python/kernel_tests/BUILD | 46 +++--- ..._test.py => interleave_dataset_op_test.py} | 136 ++++++++++++++---- tensorflow/contrib/data/python/ops/BUILD | 2 +- .../ops/{sloppy_ops.py => interleave_ops.py} | 66 ++++++++- tensorflow/core/kernels/BUILD | 6 +- .../core/kernels/map_and_batch_dataset_op.cc | 2 +- ...p.cc => parallel_interleave_dataset_op.cc} | 78 +++++++--- .../core/kernels/prefetch_dataset_op.cc | 1 - .../core/ops/compat/ops_history.v1.pbtxt | 89 ------------ tensorflow/core/ops/dataset_ops.cc | 3 +- 11 files changed, 252 insertions(+), 179 deletions(-) rename tensorflow/contrib/data/python/kernel_tests/{sloppy_transformation_dataset_op_test.py => interleave_dataset_op_test.py} (84%) rename tensorflow/contrib/data/python/ops/{sloppy_ops.py => interleave_ops.py} (67%) rename tensorflow/core/kernels/{sloppy_interleave_dataset_op.cc => parallel_interleave_dataset_op.cc} (84%) diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index e0aab1cd83..6c46acf204 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -50,6 +50,7 @@ from tensorflow.contrib.data.python.ops.dataset_ops import get_single_element from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset from tensorflow.contrib.data.python.ops.error_ops import ignore_errors from tensorflow.contrib.data.python.ops.grouping import group_by_window +from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator from tensorflow.contrib.data.python.ops.readers import FixedLengthRecordDataset from tensorflow.contrib.data.python.ops.readers import read_batch_features @@ -57,7 +58,6 @@ from tensorflow.contrib.data.python.ops.readers import SqlDataset from tensorflow.contrib.data.python.ops.readers import TextLineDataset from tensorflow.contrib.data.python.ops.readers import TFRecordDataset from tensorflow.contrib.data.python.ops.resampling import rejection_resample -from tensorflow.contrib.data.python.ops.sloppy_ops import sloppy_interleave from tensorflow.python.data.ops.iterator_ops import Iterator # pylint: enable=unused-import diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index c310e79741..ff59e80b79 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -143,6 +143,29 @@ py_test( ], ) +py_test( + name = "interleave_dataset_op_test", + size = "small", + srcs = ["interleave_dataset_op_test.py"], + srcs_version = "PY2AND3", + tags = [ + "manual", # b/67958761 + ], + deps = [ + "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:math_ops", + "//tensorflow/python:script_ops", + "//tensorflow/python:training", + "//third_party/py/numpy", + ], +) + py_test( name = "iterator_ops_cluster_test", size = "small", @@ -352,29 +375,6 @@ py_test( ], ) -py_test( - name = "sloppy_transformation_dataset_op_test", - size = "small", - srcs = ["sloppy_transformation_dataset_op_test.py"], - srcs_version = "PY2AND3", - tags = [ - "manual", # b/67958761 - ], - deps = [ - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//tensorflow/python:math_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", - ], -) - py_test( name = "sql_dataset_op_test", size = "small", diff --git a/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py similarity index 84% rename from tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py rename to tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py index 880e01dc06..0aa9ea88de 100644 --- a/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py @@ -25,7 +25,7 @@ import time from six.moves import zip_longest from tensorflow.contrib.data.python.ops import dataset_ops -from tensorflow.contrib.data.python.ops import sloppy_ops +from tensorflow.contrib.data.python.ops import interleave_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.ops import array_ops @@ -34,12 +34,13 @@ from tensorflow.python.ops import script_ops from tensorflow.python.platform import test -class SloppyInterleaveDatasetTest(test.TestCase): +class ParallelInterleaveDatasetTest(test.TestCase): def setUp(self): self.input_values = array_ops.placeholder(dtypes.int64, shape=[None]) self.cycle_length = array_ops.placeholder(dtypes.int64, shape=[]) self.block_length = array_ops.placeholder(dtypes.int64, shape=[]) + self.sloppy = array_ops.placeholder(dtypes.bool, shape=[]) self.repeat_count = 2 @@ -69,9 +70,9 @@ class SloppyInterleaveDatasetTest(test.TestCase): self.dataset = (dataset_ops.Dataset.from_tensor_slices(self.input_values) .repeat(self.repeat_count).apply( - sloppy_ops.sloppy_interleave( + interleave_ops.parallel_interleave( interleave_fn, self.cycle_length, - self.block_length))) + self.block_length, self.sloppy))) self.iterator = self.dataset.make_initializable_iterator() self.init_op = self.iterator.initializer self.next_element = self.iterator.get_next() @@ -161,7 +162,7 @@ class SloppyInterleaveDatasetTest(test.TestCase): for i in range(4, 7): self.write_coordination_events[i].set() - def testSingleThreaded(self): + def _testSingleThreaded(self, sloppy=False): # cycle_length=1,block_length=1 acts like `Dataset.interleave()` and # `Dataset.flat_map()` and is single-threaded. No synchronization required. with self.test_session() as sess: @@ -171,7 +172,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 1, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy }) for expected_element in self._interleave( @@ -182,7 +184,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContention(self): + def testSingleThreaded(self): + self._testSingleThreaded() + + def testSingleThreadedSloppy(self): + self._testSingleThreaded(sloppy=True) + + def _testTwoThreadsNoContention(self, sloppy=False): # num_threads > 1. # Explicit coordination should result in `Dataset.interleave()` behavior with self.test_session() as sess: @@ -193,7 +201,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -211,11 +220,20 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContentionWithRaces(self): + def testTwoThreadsNoContention(self): + self._testTwoThreadsNoContention() + + def testTwoThreadsNoContentionSloppy(self): + self._testTwoThreadsNoContention(sloppy=True) + + def _testTwoThreadsNoContentionWithRaces(self, sloppy=False): """Tests where all the workers race in producing elements. Note: this is in contrast with the prevous test which carefully sequences the execution of the map functions. + + Args: + sloppy: Whether to be sloppy or not. """ with self.test_session() as sess: self._clear_coordination_events() @@ -225,7 +243,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy, }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -247,7 +266,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContentionBlockLength(self): + def testTwoThreadsNoContentionWithRaces(self): + self._testTwoThreadsNoContentionWithRaces() + + def testTwoThreadsNoContentionWithRacesSloppy(self): + self._testTwoThreadsNoContentionWithRaces(sloppy=True) + + def _testTwoThreadsNoContentionBlockLength(self, sloppy=False): # num_threads > 1. # Explicit coordination should result in `Dataset.interleave()` behavior with self.test_session() as sess: @@ -258,7 +283,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 2 + self.block_length: 2, + self.sloppy: sloppy }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -276,11 +302,21 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContentionWithRacesAndBlocking(self): + def testTwoThreadsNoContentionBlockLength(self): + self._testTwoThreadsNoContentionBlockLength() + + def testTwoThreadsNoContentionBlockLengthSloppy(self): + self._testTwoThreadsNoContentionBlockLength(sloppy=True) + + def _testTwoThreadsNoContentionWithRacesAndBlocking(self, sloppy=False): """Tests where all the workers race in producing elements. Note: this is in contrast with the prevous test which carefully sequences the execution of the map functions. + + + Args: + sloppy: Whether to be sloppy or not. """ with self.test_session() as sess: self._clear_coordination_events() @@ -290,7 +326,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 2 + self.block_length: 2, + self.sloppy: sloppy }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -312,7 +349,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testEmptyInput(self): + def testTwoThreadsNoContentionWithRacesAndBlocking(self): + self._testTwoThreadsNoContentionWithRacesAndBlocking() + + def testTwoThreadsNoContentionWithRacesAndBlockingSloppy(self): + self._testTwoThreadsNoContentionWithRacesAndBlocking(sloppy=True) + + def _testEmptyInput(self, sloppy=False): with self.test_session() as sess: # Empty input. self._clear_coordination_events() @@ -321,12 +364,19 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [], self.cycle_length: 2, - self.block_length: 3 + self.block_length: 3, + self.sloppy: sloppy }) with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testNonEmptyInputIntoEmptyOutputs(self): + def testEmptyInput(self): + self._testEmptyInput() + + def testEmptyInputSloppy(self): + self._testEmptyInput(sloppy=True) + + def _testNonEmptyInputIntoEmptyOutputs(self, sloppy=False): # Non-empty input leading to empty output. with self.test_session() as sess: self._clear_coordination_events() @@ -335,12 +385,19 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [0, 0, 0], self.cycle_length: 2, - self.block_length: 3 + self.block_length: 3, + self.sloppy: sloppy }) with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testPartiallyEmptyOutputs(self): + def testNonEmptyInputIntoEmptyOutputs(self): + self._testNonEmptyInputIntoEmptyOutputs() + + def testNonEmptyInputIntoEmptyOutputsSloppy(self): + self._testNonEmptyInputIntoEmptyOutputs(sloppy=True) + + def _testPartiallyEmptyOutputs(self, sloppy=False): # Mixture of non-empty and empty interleaved datasets. with self.test_session() as sess: self._clear_coordination_events() @@ -350,7 +407,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 0, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy, }) for i, expected_element in enumerate( self._interleave([[4] * 4, [], [6] * 6] * self.repeat_count, 2, 1)): @@ -367,7 +425,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testDelayedOutput(self): + def testPartiallyEmptyOutputs(self): + self._testPartiallyEmptyOutputs() + + def testPartiallyEmptyOutputsSloppy(self): + self._testPartiallyEmptyOutputs(sloppy=True) + + def testDelayedOutputSloppy(self): # Explicitly control the sequence of events to ensure we correctly avoid # head-of-line blocking. with self.test_session() as sess: @@ -377,7 +441,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: True, }) mis_ordering = [ @@ -391,7 +456,7 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testBlockLengthWithContention(self): + def testBlockLengthWithContentionSloppy(self): with self.test_session() as sess: self._clear_coordination_events() done_first_event = False @@ -400,7 +465,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 3 + self.block_length: 3, + self.sloppy: True }) # Test against a generating sequence that differs from the uncontended # case, in order to prove sloppy correctness. @@ -422,7 +488,7 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testEarlyExit(self): + def _testEarlyExit(self, sloppy=False): # Exiting without consuming all input should not block with self.test_session() as sess: self._clear_coordination_events() @@ -431,7 +497,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 3, - self.block_length: 2 + self.block_length: 2, + self.sloppy: sloppy }) for i in range(4, 7): self.write_coordination_events[i].set() @@ -445,7 +512,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): self.read_coordination_events[i].acquire() self.write_coordination_events[i].set() - def testTooManyReaders(self): + def testEarlyExit(self): + self._testEarlyExit() + + def testEarlyExitSloppy(self): + self._testEarlyExit(sloppy=True) + + def _testTooManyReaders(self, sloppy=False): def interleave_fn(x): dataset = dataset_ops.Dataset.from_tensors(x) @@ -455,8 +528,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): dataset = dataset_ops.Dataset.from_tensor_slices([4, 5, 6]) dataset = dataset.repeat(self.repeat_count) dataset = dataset.apply( - sloppy_ops.sloppy_interleave(interleave_fn, cycle_length=16, - block_length=2)) + interleave_ops.parallel_interleave( + interleave_fn, cycle_length=16, block_length=2, sloppy=sloppy)) iterator = dataset.make_one_shot_iterator() with self.test_session() as sess: @@ -468,6 +541,11 @@ class SloppyInterleaveDatasetTest(test.TestCase): [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 2) self.assertItemsEqual(output_values, expected_values) + def testTooManyReaders(self): + self._testTooManyReaders() + + def testTooManyReadersSloppy(self): + self._testTooManyReaders(sloppy=True) if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index a6eb50014a..e0730488a1 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -60,9 +60,9 @@ py_library( "enumerate_ops.py", "error_ops.py", "grouping.py", + "interleave_ops.py", "resampling.py", "scan_ops.py", - "sloppy_ops.py", ], srcs_version = "PY2AND3", deps = [ diff --git a/tensorflow/contrib/data/python/ops/sloppy_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py similarity index 67% rename from tensorflow/contrib/data/python/ops/sloppy_ops.py rename to tensorflow/contrib/data/python/ops/interleave_ops.py index 4f3da4320c..74a919c1ff 100644 --- a/tensorflow/contrib/data/python/ops/sloppy_ops.py +++ b/tensorflow/contrib/data/python/ops/interleave_ops.py @@ -23,14 +23,16 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.util import deprecation -class SloppyInterleaveDataset(dataset_ops.Dataset): +class ParallelInterleaveDataset(dataset_ops.Dataset): """A `Dataset` that maps a function over its input and flattens the result.""" - def __init__(self, input_dataset, map_func, cycle_length, block_length): - """See `tf.contrib.data.sloppy_interleave()` for details.""" - super(SloppyInterleaveDataset, self).__init__() + def __init__(self, input_dataset, map_func, cycle_length, block_length, + sloppy): + """See `tf.contrib.data.parallel_interleave()` for details.""" + super(ParallelInterleaveDataset, self).__init__() self._input_dataset = input_dataset @function.Defun(*nest.flatten(input_dataset.output_types)) @@ -62,13 +64,16 @@ class SloppyInterleaveDataset(dataset_ops.Dataset): cycle_length, dtype=dtypes.int64, name="cycle_length") self._block_length = ops.convert_to_tensor( block_length, dtype=dtypes.int64, name="block_length") + self._sloppy = ops.convert_to_tensor( + sloppy, dtype=dtypes.bool, name="sloppy") def _as_variant_tensor(self): - return gen_dataset_ops.sloppy_interleave_dataset( + return gen_dataset_ops.parallel_interleave_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._map_func.captured_inputs, self._cycle_length, self._block_length, + self._sloppy, f=self._map_func, output_types=nest.flatten(self.output_types), output_shapes=nest.flatten(self.output_shapes)) @@ -82,6 +87,53 @@ class SloppyInterleaveDataset(dataset_ops.Dataset): return self._output_types +def parallel_interleave(map_func, cycle_length, block_length=1, sloppy=False): + """A parallel version of the `Dataset.interleave()` transformation. + + `parallel_interleave()` maps `map_func` across its input to produce nested + datasets, and outputs their elements interleaved. Unlike + @{tf.data.Dataset.interleave}, it gets elements from `cycle_length` nested + datasets in parallel, which increases the throughput, especially in the + presence of stragglers. Furthermore, the `sloppy` argument can be used to + improve performance, by relaxing the requirement that the outputs are produced + in a deterministic order, and allowing the implementation to skip over nested + datasets whose elements are not readily available when requested. + + Example usage: + + ```python + # Preprocess 4 files concurrently. + filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords") + dataset = filenames.apply( + tf.contrib.data.parallel_interleave( + lambda filename: tf.data.TFRecordDataset(filename), + cycle_length=4)) + ``` + + WARNING: If `sloppy` is `True`, the order of produced elements is not + deterministic. + + Args: + map_func: A function mapping a nested structure of tensors to a `Dataset`. + cycle_length: The number of threads to interleave from in parallel. + block_length: The number of consecutive elements to pull from a thread + before advancing to the next thread. + sloppy: If false, elements are produced in deterministic order. Otherwise, + the implementation is allowed, for the sake of expediency, to produce + elements in a non-deterministic order. + + Returns: + A `Dataset` transformation function, which can be passed to + @{tf.data.Dataset.apply}. + """ + def _apply_fn(dataset): + return ParallelInterleaveDataset( + dataset, map_func, cycle_length, block_length, sloppy) + return _apply_fn + + +@deprecation.deprecated( + None, "Use `tf.contrib.data.parallel_interleave(..., sloppy=True)`.") def sloppy_interleave(map_func, cycle_length, block_length=1): """A non-deterministic version of the `Dataset.interleave()` transformation. @@ -132,6 +184,6 @@ def sloppy_interleave(map_func, cycle_length, block_length=1): @{tf.data.Dataset.apply}. """ def _apply_fn(dataset): - return SloppyInterleaveDataset( - dataset, map_func, cycle_length, block_length) + return ParallelInterleaveDataset( + dataset, map_func, cycle_length, block_length, sloppy=True) return _apply_fn diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 0274f87ec6..2aef1e3560 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -5924,8 +5924,8 @@ tf_kernel_library( ) tf_kernel_library( - name = "sloppy_interleave_dataset_op", - srcs = ["sloppy_interleave_dataset_op.cc"], + name = "parallel_interleave_dataset_op", + srcs = ["parallel_interleave_dataset_op.cc"], deps = [ ":captured_function", ":dataset", @@ -6162,6 +6162,7 @@ tf_kernel_library( ":map_and_batch_dataset_op", ":map_dataset_op", ":padded_batch_dataset_op", + ":parallel_interleave_dataset_op", ":parallel_map_dataset_op", ":prefetch_dataset_op", ":range_dataset_op", @@ -6170,7 +6171,6 @@ tf_kernel_library( ":scan_dataset_op", ":shuffle_dataset_op", ":skip_dataset_op", - ":sloppy_interleave_dataset_op", ":sparse_tensor_slice_dataset_op", ":sql_dataset_ops", ":take_dataset_op", diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc index f9f68a5418..620efdb778 100644 --- a/tensorflow/core/kernels/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc @@ -336,7 +336,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { const DataTypeVector output_types_; const std::vector output_shapes_; const std::unique_ptr captured_func_; - const Eigen::ThreadPoolDevice* device_; // not owned + const Eigen::ThreadPoolDevice* device_; // not owned }; const int graph_def_version_; diff --git a/tensorflow/core/kernels/sloppy_interleave_dataset_op.cc b/tensorflow/core/kernels/parallel_interleave_dataset_op.cc similarity index 84% rename from tensorflow/core/kernels/sloppy_interleave_dataset_op.cc rename to tensorflow/core/kernels/parallel_interleave_dataset_op.cc index 8f9f48700c..56942a5c01 100644 --- a/tensorflow/core/kernels/sloppy_interleave_dataset_op.cc +++ b/tensorflow/core/kernels/parallel_interleave_dataset_op.cc @@ -17,12 +17,11 @@ limitations under the License. #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/kernels/captured_function.h" #include "tensorflow/core/kernels/dataset_utils.h" #include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/random/random.h" -#include "tensorflow/core/kernels/captured_function.h" - namespace tensorflow { namespace { @@ -30,9 +29,9 @@ namespace { // See documentation in ../ops/dataset_ops.cc for a high-level // description of the following op. -class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { +class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel { public: - explicit SloppyInterleaveDatasetOp(OpKernelConstruction* ctx) + explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx) : UnaryDatasetOpKernel(ctx), graph_def_version_(ctx->graph_def_version()) { OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_)); @@ -62,13 +61,16 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { OP_REQUIRES(ctx, block_length > 0, errors::InvalidArgument("`block_length` must be > 0")); + bool sloppy; + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "sloppy", &sloppy)); + std::unique_ptr captured_func; OP_REQUIRES_OK(ctx, CapturedFunction::Create(ctx, func_, graph_def_version_, std::move(other_arguments), &captured_func)); *output = new Dataset(input, std::move(captured_func), cycle_length, - block_length, output_types_, output_shapes_); + block_length, sloppy, output_types_, output_shapes_); } private: @@ -76,12 +78,13 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { public: Dataset(const DatasetBase* input, std::unique_ptr captured_func, int64 cycle_length, - int64 block_length, const DataTypeVector& output_types, + int64 block_length, bool sloppy, const DataTypeVector& output_types, const std::vector& output_shapes) : input_(input), captured_func_(std::move(captured_func)), cycle_length_(cycle_length), block_length_(block_length), + sloppy_(sloppy), output_types_(output_types), output_shapes_(output_shapes) { input_->Ref(); @@ -91,8 +94,8 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { std::unique_ptr MakeIterator( const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::SloppyInterleave")})); + return std::unique_ptr(new Iterator( + {this, strings::StrCat(prefix, "::ParallelInterleave")})); } const DataTypeVector& output_dtypes() const override { @@ -103,7 +106,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { } string DebugString() override { - return "SloppyInterleaveDatasetOp::Dataset"; + return "ParallelInterleaveDatasetOp::Dataset"; } private: @@ -131,16 +134,24 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { bool* end_of_sequence) override { mutex_lock l(mu_); TF_RETURN_IF_ERROR(EnsureWorkerThreadsStarted(ctx)); - // Search for available items, blocking if necessary. + const int64 num_workers = worker_threads_.size(); + if (num_workers == 0) { + *end_of_sequence = true; + return Status::OK(); + } while (!cancelled_) { - for (size_t i = 0; i < dataset()->cycle_length_; ++i) { - size_t index = (next_index_ + i) % dataset()->cycle_length_; + // Wait for an item to become available, blocking if necessary. If we + // are allowed to be sloppy, we can skip over input datasets that do + // not have an item readily available. + const int64 n = dataset()->sloppy_ ? num_workers : 1LL; + for (int64 i = 0; i < n; ++i) { + int64 index = (next_index_ + i) % num_workers; if (output_elements_[index].is_produced) { next_index_ = index; if (i == 0) { block_count_++; if (block_count_ == dataset()->block_length_) { - next_index_ = (index + 1) % dataset()->cycle_length_; + next_index_ = (index + 1) % num_workers; block_count_ = 0; } } else { @@ -150,7 +161,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { if (output_elements_[index].end_of_sequence) { output_elements_[index].is_produced = false; output_elements_[index].cond_var.notify_one(); - next_index_ = (index + 1) % dataset()->cycle_length_; + next_index_ = (index + 1) % num_workers; block_count_ = 0; i = -1; // Restart the inner loop continue; @@ -174,11 +185,21 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { *end_of_sequence = true; return Status::OK(); } + + // If we are not allowed to be sloppy and + // `worker_threads_[next_index]` has finished, advance `next_index`. + if (!dataset()->sloppy_ && worker_threads_[next_index_].finished) { + next_index_ = (next_index_ + 1) % num_workers; + continue; + } + // No values available; wait until woken up. + // TODO(jsimsa): Use slot-specific condition variable for + // coordination of elements consumption. cond_var_.wait(l); } return errors::Cancelled( - "SloppyInterleaveDatasetOp::Dataset::Iterator::GetNext"); + "ParallelInterleaveDatasetOp::Dataset::Iterator::GetNext"); } private: @@ -201,6 +222,16 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { condition_variable cond_var; }; + struct ThreadStatus { + // The underlying thread uses `finished` to communicate to the producer + // that it has finished. + bool finished = false; + // The underlying thread object. + std::unique_ptr thread; + + explicit ThreadStatus(Thread* thread) : thread(thread) {} + }; + Status EnsureWorkerThreadsStarted(IteratorContext* ctx) EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (worker_threads_.empty()) { @@ -220,11 +251,10 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { std::unique_ptr itr; TF_RETURN_IF_ERROR(dataset::MakeIteratorFromInputElement( ctx, args, i, dataset()->captured_func_.get(), prefix(), &itr)); - worker_threads_.emplace_back( - std::unique_ptr(ctx->env()->StartThread( - {}, "worker_thread", - std::bind(&Iterator::WorkerThread, this, - new IteratorContext(*ctx), i, itr.release())))); + worker_threads_.emplace_back(ctx->env()->StartThread( + {}, "worker_thread", + std::bind(&Iterator::WorkerThread, this, + new IteratorContext(*ctx), i, itr.release()))); num_active_threads_ = i + 1; } } @@ -264,6 +294,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { std::unique_ptr out_iterator(out_iterator_ptr); auto cleanup = gtl::MakeCleanup([this, thread_index] { mutex_lock l(mu_); + worker_threads_[thread_index].finished = true; num_active_threads_--; cond_var_.notify_all(); }); @@ -345,13 +376,14 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { // Pointers to the worker threads. This must be last to ensure the // threads have exited before any other members are deallocated. // TODO(b/65178177): Avoid allocating additional threads. - std::vector> worker_threads_ GUARDED_BY(mu_); + std::vector worker_threads_ GUARDED_BY(mu_); }; const DatasetBase* const input_; const std::unique_ptr captured_func_; const int64 cycle_length_; const int64 block_length_; + const bool sloppy_; const DataTypeVector output_types_; const std::vector output_shapes_; }; @@ -362,8 +394,8 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { NameAttrList func_; }; -REGISTER_KERNEL_BUILDER(Name("SloppyInterleaveDataset").Device(DEVICE_CPU), - SloppyInterleaveDatasetOp); +REGISTER_KERNEL_BUILDER(Name("ParallelInterleaveDataset").Device(DEVICE_CPU), + ParallelInterleaveDatasetOp); } // namespace diff --git a/tensorflow/core/kernels/prefetch_dataset_op.cc b/tensorflow/core/kernels/prefetch_dataset_op.cc index 8c846919c4..a7aac508eb 100644 --- a/tensorflow/core/kernels/prefetch_dataset_op.cc +++ b/tensorflow/core/kernels/prefetch_dataset_op.cc @@ -59,7 +59,6 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel { Dataset(const DatasetBase* input, int64 buffer_size, IteratorContext::Params ctx_params) : input_(input), - buffer_size_(buffer_size), ctx_params_(std::move(ctx_params)) { input_->Ref(); diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 076c7bea1a..a691065075 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -32629,95 +32629,6 @@ op { } } } -op { - name: "SloppyInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} -op { - name: "SloppyInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "Softmax" input_arg { diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 8b77e3f9f0..f512213964 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -285,11 +285,12 @@ f: A function mapping elements of `input_dataset`, concatenated with `output_types` and `output_shapes`. )doc"); -REGISTER_OP("SloppyInterleaveDataset") +REGISTER_OP("ParallelInterleaveDataset") .Input("input_dataset: variant") .Input("other_arguments: Targuments") .Input("cycle_length: int64") .Input("block_length: int64") + .Input("sloppy: bool") .Output("handle: variant") .Attr("f: func") .Attr("Targuments: list(type) >= 0") -- GitLab From 16538dab77bac6b79a05aa91c43e53227b45c945 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 27 Oct 2017 10:49:59 -0700 Subject: [PATCH 464/573] Saves summaries in the mnist example. PiperOrigin-RevId: 173690505 --- .../contrib/eager/python/metrics_impl.py | 2 +- .../contrib/eager/python/metrics_test.py | 26 ++----- tensorflow/contrib/summary/summary_ops.py | 74 +++++++++++++------ tensorflow/python/training/training_util.py | 13 +++- 4 files changed, 72 insertions(+), 43 deletions(-) diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 795dff548f..2ba653af4a 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -90,7 +90,7 @@ class Metric(object): # We create the variable scope now to get the unique name that will # be used as a variable prefix when build() calls add_variable(). with variable_scope.variable_scope( - None, default_name=scope_name, use_resource=True, reuse=False) as scope: + scope_name, use_resource=True, reuse=False) as scope: pos = scope.name.rfind(scope_name) self._name = name + scope.name[pos + len(scope_name):] self._scope = scope diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index a8377a0660..336ce9d307 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -120,24 +120,18 @@ class MetricsTest(test.TestCase): # Verify two metrics with the same class and name don't # accidentally share state. m1 = metrics.Mean() - m2 = metrics.Mean() m1(0) - m2(2) - self.assertEqual(0, m1.result().numpy()) - self.assertEqual(2, m2.result().numpy()) - self.assertNotEqual(m1.name, m2.name) + with self.assertRaises(ValueError): + m2 = metrics.Mean() + m2(2) def testNamesWithSpaces(self): # Verify two metrics with the same class and name don't # accidentally share state. m1 = metrics.Mean("has space") - m2 = metrics.Mean("has space") - m2(2) m1(0) self.assertEqual(m1.name, "has space") self.assertEqual(m1.numer.name, "has_space/numer:0") - self.assertEqual(m2.name, "has space_1") - self.assertEqual(m2.numer.name, "has_space_1/numer:0") def testGraph(self): with context.graph_mode(), self.test_session() as sess: @@ -158,16 +152,12 @@ class MetricsTest(test.TestCase): def testTwoMeansGraph(self): # Verify two metrics with the same class and name don't # accidentally share state. - with context.graph_mode(), self.test_session() as sess: + with context.graph_mode(): m1 = metrics.Mean() - m2 = metrics.Mean() - accumulate1 = m1(0) - accumulate2 = m2(2) - m1.init_variables().run() - m2.init_variables().run() - sess.run([accumulate1, accumulate2]) - self.assertEqual(0, m1.result().eval()) - self.assertEqual(2, m2.result().eval()) + m1(0) + with self.assertRaises(ValueError): + m2 = metrics.Mean() + m2(2) if __name__ == "__main__": diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index b32b093675..9c71bf7740 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -26,6 +26,8 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import summary_op_util from tensorflow.python.training import training_util @@ -57,7 +59,8 @@ def record_summaries_every_n_global_steps(n): """Sets the should_record_summaries Tensor to true if global_step % n == 0.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) old = collection_ref[:] - collection_ref[:] = [training_util.get_global_step() % n == 0] + with ops.device("cpu:0"): + collection_ref[:] = [math_ops.equal(training_util.get_global_step() % n, 0)] yield collection_ref[:] = old @@ -97,13 +100,17 @@ class SummaryWriter(object): @tf_contextlib.contextmanager def as_default(self): - old = context.context().summary_writer_resource - context.context().summary_writer_resource = self._resource - yield - # Flushes the summary writer in eager mode or in graph functions, but not in - # legacy graph mode (you're on your own there). - gen_summary_ops.flush_summary_writer(self._resource) - context.context().summary_writer_resource = old + if self._resource is None: + yield + else: + old = context.context().summary_writer_resource + context.context().summary_writer_resource = self._resource + yield + # Flushes the summary writer in eager mode or in graph functions, but not + # in legacy graph mode (you're on your own there). + with ops.device("cpu:0"): + gen_summary_ops.flush_summary_writer(self._resource) + context.context().summary_writer_resource = old def create_summary_file_writer(logdir, @@ -111,21 +118,40 @@ def create_summary_file_writer(logdir, flush_secs=None, filename_suffix=None, name=None): - """Creates a summary file writer in the current context.""" - if max_queue is None: - max_queue = constant_op.constant(10) - if flush_secs is None: - flush_secs = constant_op.constant(120) - if filename_suffix is None: - filename_suffix = constant_op.constant("") - resource = gen_summary_ops.summary_writer(shared_name=name) - # TODO(apassos) ensure the initialization op runs when in graph mode; consider - # calling session.run here. - ops.add_to_collection( - _SUMMARY_WRITER_INIT_COLLECTION_NAME, - gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, - flush_secs, filename_suffix)) - return SummaryWriter(resource) + """Creates a summary file writer in the current context. + + Args: + logdir: a string, or None. If a string, creates a summary file writer + which writes to the directory named by the string. If None, returns + a mock object which acts like a summary writer but does nothing, + useful to use as a context manager. + max_queue: the largest number of summaries to keep in a queue; will + flush once the queue gets bigger than this. + flush_secs: the largest interval (in seconds) between flushes. + filename_suffix: optional suffix for the event file name. + name: name for the summary writer. + + Returns: + Either a summary writer or an empty object which can be used as a + summary writer. + """ + if logdir is None: + return SummaryWriter(None) + with ops.device("cpu:0"): + if max_queue is None: + max_queue = constant_op.constant(10) + if flush_secs is None: + flush_secs = constant_op.constant(120) + if filename_suffix is None: + filename_suffix = constant_op.constant("") + resource = gen_summary_ops.summary_writer(shared_name=name) + # TODO(apassos) ensure the initialization op runs when in graph mode; + # consider calling session.run here. + ops.add_to_collection( + _SUMMARY_WRITER_INIT_COLLECTION_NAME, + gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, + flush_secs, filename_suffix)) + return SummaryWriter(resource) def _nothing(): @@ -168,6 +194,8 @@ def summary_writer_function(name, tensor, function, family=None): with ops.control_dependencies([function(tag, scope)]): return constant_op.constant(True) + if context.context().summary_writer_resource is None: + return control_flow_ops.no_op() with ops.device("cpu:0"): op = utils.smart_cond( should_record_summaries(), record, _nothing, name="") diff --git a/tensorflow/python/training/training_util.py b/tensorflow/python/training/training_util.py index bdd4ca734e..89a9e12932 100644 --- a/tensorflow/python/training/training_util.py +++ b/tensorflow/python/training/training_util.py @@ -119,13 +119,24 @@ def create_global_step(graph=None): raise ValueError('"global_step" already exists.') # Create in proper graph and base name_scope. with graph.as_default() as g, g.name_scope(None): + if context.in_eager_mode(): + with ops.device('cpu:0'): + return variable_scope.get_variable( + ops.GraphKeys.GLOBAL_STEP, + shape=[], + dtype=dtypes.int64, + initializer=init_ops.zeros_initializer(), + trainable=False, + collections=[ops.GraphKeys.GLOBAL_VARIABLES, + ops.GraphKeys.GLOBAL_STEP]) return variable_scope.get_variable( ops.GraphKeys.GLOBAL_STEP, shape=[], dtype=dtypes.int64, initializer=init_ops.zeros_initializer(), trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP]) + collections=[ops.GraphKeys.GLOBAL_VARIABLES, + ops.GraphKeys.GLOBAL_STEP]) def get_or_create_global_step(graph=None): -- GitLab From 873ef2ca375f2f6a4fed55e0bcd9a6fba6e8d545 Mon Sep 17 00:00:00 2001 From: Oleg Zabluda Date: Fri, 27 Oct 2017 10:56:18 -0700 Subject: [PATCH 465/573] Fix documentation error in tf.size() - output type --- tensorflow/python/ops/array_ops.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 857cd09d56..712ae8620e 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -322,11 +322,11 @@ def size(input, name=None, out_type=dtypes.int32): Args: input: A `Tensor` or `SparseTensor`. name: A name for the operation (optional). - out_type: (Optional) The specified output type of the operation - (`int32` or `int64`). Defaults to tf.int32. + out_type: (Optional) The specified non-quantized numeric output type + of the operation. Defaults to `tf.int32`. Returns: - A `Tensor` of type `out_type`. Defaults to tf.int32. + A `Tensor` of type `out_type`. Defaults to `tf.int32`. @compatibility(numpy) Equivalent to np.size() @@ -343,11 +343,11 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32): input: A `Tensor` or `SparseTensor`. name: A name for the operation (optional). optimize: if true, encode the size as a constant when possible. - out_type: (Optional) The specified output type of the operation - (`int32` or `int64`). Defaults to tf.int32. + out_type: (Optional) The specified non-quantized numeric output type + of the operation. Defaults to `tf.int32`. Returns: - A `Tensor` of type `out_type`. + A `Tensor` of type `out_type`. Defaults to `tf.int32`. """ with ops.name_scope(name, "Size", [input]) as name: if isinstance(input, (sparse_tensor.SparseTensor, -- GitLab From 97484a4d90e6a1bbd8784bffd3b3af41d777d6bd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 10:51:31 -0700 Subject: [PATCH 466/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173690751 --- .../core/ops/compat/ops_history.v1.pbtxt | 48 +++++++++ tensorflow/core/ops/ops.pbtxt | 98 ++++++++++--------- 2 files changed, 99 insertions(+), 47 deletions(-) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index a691065075..4d00694707 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -20723,6 +20723,54 @@ op { type: "type" } } +op { + name: "ParallelInterleaveDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "cycle_length" + type: DT_INT64 + } + input_arg { + name: "block_length" + type: DT_INT64 + } + input_arg { + name: "sloppy" + type: DT_BOOL + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} op { name: "ParallelMapDataset" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 0a590fef00..e43ee0d986 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -16181,6 +16181,57 @@ op { summary: "Interleave the values from the `data` tensors into a single tensor." description: "Builds a merged tensor such that\n\n```python\n merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]\n```\n\nFor example, if each `indices[m]` is scalar or vector, we have\n\n```python\n # Scalar indices:\n merged[indices[m], ...] = data[m][...]\n\n # Vector indices:\n merged[indices[m][i], ...] = data[m][i, ...]\n```\n\nEach `data[i].shape` must start with the corresponding `indices[i].shape`,\nand the rest of `data[i].shape` must be constant w.r.t. `i`. That is, we\nmust have `data[i].shape = indices[i].shape + constant`. In terms of this\n`constant`, the output shape is\n\n merged.shape = [max(indices)] + constant\n\nValues may be merged in parallel, so if an index appears in both `indices[m][i]`\nand `indices[n][j]`, the result may be invalid. This differs from the normal\nDynamicStitch operator that defines the behavior in that case.\n\nFor example:\n\n```python\n indices[0] = 6\n indices[1] = [4, 1]\n indices[2] = [[5, 2], [0, 3]]\n data[0] = [61, 62]\n data[1] = [[41, 42], [11, 12]]\n data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]\n merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],\n [51, 52], [61, 62]]\n```\n\nThis method can be used to merge partitions created by `dynamic_partition`\nas illustrated on the following example:\n\n```python\n # Apply function (increments x_i) on elements for which a certain condition\n # apply (x_i != -1 in this example).\n x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])\n condition_mask=tf.not_equal(x,tf.constant(-1.))\n partitioned_data = tf.dynamic_partition(\n x, tf.cast(condition_mask, tf.int32) , 2)\n partitioned_data[1] = partitioned_data[1] + 1.0\n condition_indices = tf.dynamic_partition(\n tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)\n x = tf.dynamic_stitch(condition_indices, partitioned_data)\n # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain\n # unchanged.\n```\n\n
\n\n
" } +op { + name: "ParallelInterleaveDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "cycle_length" + type: DT_INT64 + } + input_arg { + name: "block_length" + type: DT_INT64 + } + input_arg { + name: "sloppy" + type: DT_BOOL + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + description: "A function mapping elements of `input_dataset`, concatenated with\n`other_arguments`, to a Dataset variant that contains elements matching\n`output_types` and `output_shapes`." + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`." + description: "The resulting dataset is similar to the `InterleaveDataset`, with the exception\nthat if retrieving the next value from a dataset would cause the requester to\nblock, it will skip that input dataset. This dataset is especially useful\nwhen loading data from a variable-latency datastores (e.g. HDFS, GCS), as it\nallows the training step to proceed so long as some data is available.\n\n!! WARNING !! This dataset is not deterministic!" +} op { name: "ParallelMapDataset" input_arg { @@ -25772,53 +25823,6 @@ op { summary: "Return a slice from \'input\'." description: "The output tensor is a tensor with dimensions described by \'size\'\nwhose values are extracted from \'input\' starting at the offsets in\n\'begin\'.\n\n*Requirements*:\n 0 <= begin[i] <= begin[i] + size[i] <= Di for i in [0, n)" } -op { - name: "SloppyInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - description: "A function mapping elements of `input_dataset`, concatenated with\n`other_arguments`, to a Dataset variant that contains elements matching\n`output_types` and `output_shapes`." - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`." - description: "The resulting dataset is similar to the `InterleaveDataset`, with the exception\nthat if retrieving the next value from a dataset would cause the requester to\nblock, it will skip that input dataset. This dataset is especially useful\nwhen loading data from a variable-latency datastores (e.g. HDFS, GCS), as it\nallows the training step to proceed so long as some data is available.\n\n!! WARNING !! This dataset is not deterministic!" -} op { name: "Softmax" input_arg { -- GitLab From 32bcf46f13a1ad158f366523289d06e6d00642b6 Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Fri, 27 Oct 2017 11:37:28 -0700 Subject: [PATCH 467/573] internal PiperOrigin-RevId: 173697389 --- tensorflow/contrib/kernel_methods/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/kernel_methods/BUILD b/tensorflow/contrib/kernel_methods/BUILD index ae1402b0e6..a2f320ab11 100644 --- a/tensorflow/contrib/kernel_methods/BUILD +++ b/tensorflow/contrib/kernel_methods/BUILD @@ -64,6 +64,7 @@ py_test( name = "kernel_estimators_test", srcs = ["python/kernel_estimators_test.py"], srcs_version = "PY2AND3", + tags = ["notsan"], deps = [ ":kernel_methods", "//tensorflow/contrib/layers:layers_py", -- GitLab From 73155f56a30c75eb54d95ab0d51ab8b5c8fb02c9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 12:12:31 -0700 Subject: [PATCH 468/573] [TF:XLA] Small code cleanup. Re-alphabetized. PiperOrigin-RevId: 173702336 --- tensorflow/compiler/xla/service/hlo_opcode.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc index 2f2263f70d..d94c4da5ea 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.cc +++ b/tensorflow/compiler/xla/service/hlo_opcode.cc @@ -33,6 +33,8 @@ string HloOpcodeString(HloOpcode opcode) { return "abs"; case HloOpcode::kAdd: return "add"; + case HloOpcode::kAnd: + return "and"; case HloOpcode::kAtan2: return "atan2"; case HloOpcode::kBatchNormTraining: @@ -103,12 +105,6 @@ string HloOpcodeString(HloOpcode opcode) { return "less-than-or-equal-to"; case HloOpcode::kLog: return "log"; - case HloOpcode::kAnd: - return "and"; - case HloOpcode::kOr: - return "or"; - case HloOpcode::kNot: - return "not"; case HloOpcode::kLt: return "less-than"; case HloOpcode::kMap: @@ -123,6 +119,10 @@ string HloOpcodeString(HloOpcode opcode) { return "not-equal-to"; case HloOpcode::kNegate: return "negate"; + case HloOpcode::kNot: + return "not"; + case HloOpcode::kOr: + return "or"; case HloOpcode::kOutfeed: return "outfeed"; case HloOpcode::kPad: @@ -190,6 +190,7 @@ StatusOr StringToHloOpcode(const string& opcode_name) { static auto* opcode_map = new tensorflow::gtl::FlatMap( {{"abs", HloOpcode::kAbs}, {"add", HloOpcode::kAdd}, + {"and", HloOpcode::kAnd}, {"batch-norm-training", HloOpcode::kBatchNormTraining}, {"batch-norm-inference", HloOpcode::kBatchNormInference}, {"batch-norm-grad", HloOpcode::kBatchNormGrad}, @@ -222,16 +223,15 @@ StatusOr StringToHloOpcode(const string& opcode_name) { {"is-finite", HloOpcode::kIsFinite}, {"less-than-or-equal-to", HloOpcode::kLe}, {"log", HloOpcode::kLog}, - {"and", HloOpcode::kAnd}, - {"or", HloOpcode::kOr}, - {"not", HloOpcode::kNot}, {"less-than", HloOpcode::kLt}, {"map", HloOpcode::kMap}, {"maximum", HloOpcode::kMaximum}, {"minimum", HloOpcode::kMinimum}, {"multiply", HloOpcode::kMultiply}, + {"not", HloOpcode::kNot}, {"not-equal-to", HloOpcode::kNe}, {"negate", HloOpcode::kNegate}, + {"or", HloOpcode::kOr}, {"outfeed", HloOpcode::kOutfeed}, {"pad", HloOpcode::kPad}, {"parameter", HloOpcode::kParameter}, -- GitLab From 3d39b32b9a7833807dad037c3f57c818e9251f85 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 12:15:56 -0700 Subject: [PATCH 469/573] Fix a tfprof bug. Throws an error when the flops cannot be calculated. PiperOrigin-RevId: 173702740 --- tensorflow/python/profiler/internal/flops_registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/profiler/internal/flops_registry.py b/tensorflow/python/profiler/internal/flops_registry.py index e143501049..147711b1d9 100644 --- a/tensorflow/python/profiler/internal/flops_registry.py +++ b/tensorflow/python/profiler/internal/flops_registry.py @@ -373,6 +373,7 @@ def _max_pool_grad_flops(graph, node): kernel_area = _list_product(kernel_shape) orig_out_shape = graph_util.tensor_shape_from_node_def_name(graph, node.input[1]) + orig_out_shape.assert_is_fully_defined() max_pool_ops = kernel_area * orig_out_shape.num_elements() return ops.OpStats("flops", max_pool_ops + orig_out_shape.num_elements()) -- GitLab From 9158f974a346b4fae89044d8724eb052d466112b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 12:16:44 -0700 Subject: [PATCH 470/573] Use tf.app.run in gcs_smoke, so that the flags are explicitly parsed, instead of parsed when first accessed. PiperOrigin-RevId: 173702828 --- tensorflow/tools/gcs_test/python/gcs_smoke.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tensorflow/tools/gcs_test/python/gcs_smoke.py b/tensorflow/tools/gcs_test/python/gcs_smoke.py index 9882f75a8a..ad4cb17ae1 100644 --- a/tensorflow/tools/gcs_test/python/gcs_smoke.py +++ b/tensorflow/tools/gcs_test/python/gcs_smoke.py @@ -35,6 +35,7 @@ flags.DEFINE_integer("num_examples", 10, "Number of examples to generate") FLAGS = flags.FLAGS + def create_examples(num_examples, input_mean): """Create ExampleProto's containing data.""" ids = np.arange(num_examples).reshape([num_examples, 1]) @@ -49,6 +50,7 @@ def create_examples(num_examples, input_mean): examples.append(ex) return examples + def create_dir_test(): """Verifies file_io directory handling methods.""" @@ -122,6 +124,7 @@ def create_dir_test(): print("Deleted directory recursively %s in %s milliseconds" % ( dir_name, elapsed_ms)) + def create_object_test(): """Verifies file_io's object manipulation methods .""" starttime_ms = int(round(time.time() * 1000)) @@ -142,7 +145,8 @@ def create_object_test(): print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file creation.") elapsed_ms = int(round(time.time() * 1000)) - starttime_ms - print("Created %d files in %s milliseconds" % (len(files_to_create), elapsed_ms)) + print("Created %d files in %s milliseconds" % ( + len(files_to_create), elapsed_ms)) # Listing files of pattern1. list_files_pattern = "%s/test_file*.txt" % dir_name @@ -185,7 +189,9 @@ def create_object_test(): file_io.delete_recursively(dir_name) -if __name__ == "__main__": +def main(argv): + del argv # Unused. + # Sanity check on the GCS bucket URL. if not FLAGS.gcs_bucket_url or not FLAGS.gcs_bucket_url.startswith("gs://"): print("ERROR: Invalid GCS bucket URL: \"%s\"" % FLAGS.gcs_bucket_url) @@ -210,7 +216,7 @@ if __name__ == "__main__": # tf_record_iterator works. record_iter = tf.python_io.tf_record_iterator(input_path) read_count = 0 - for r in record_iter: + for _ in record_iter: read_count += 1 print("Read %d records using tf_record_iterator" % read_count) @@ -222,7 +228,7 @@ if __name__ == "__main__": # Verify that running the read op in a session works. print("\n=== Testing TFRecordReader.read op in a session... ===") - with tf.Graph().as_default() as g: + with tf.Graph().as_default(): filename_queue = tf.train.string_input_producer([input_path], num_epochs=1) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) @@ -249,3 +255,7 @@ if __name__ == "__main__": create_dir_test() create_object_test() + + +if __name__ == "__main__": + tf.app.run(main) -- GitLab From d7cffe9c03384189ec7509fc24d1a76f0f1241b6 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Fri, 27 Oct 2017 12:19:12 -0700 Subject: [PATCH 471/573] Adds save and restore methods to tfe.Network Save just saves the variables to a checkpoint. Restore either restores immediately or defers the restoration to variable creation time with a custom getter. PiperOrigin-RevId: 173703075 --- tensorflow/contrib/eager/python/network.py | 467 +++++++++++++++++- .../contrib/eager/python/network_test.py | 283 +++++++++++ 2 files changed, 728 insertions(+), 22 deletions(-) diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index 025d447455..5b53a597f2 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -19,11 +19,17 @@ from __future__ import division from __future__ import print_function import collections +import os import weakref +from tensorflow.python.eager import context from tensorflow.python.estimator import util as estimator_util +from tensorflow.python.framework import ops from tensorflow.python.layers import base from tensorflow.python.ops import variable_scope +from tensorflow.python.training import checkpoint_utils +from tensorflow.python.training import saver as saver_lib +from tensorflow.python.training import training_util # pylint: disable=protected-access # Explanation for protected-access disable: Network has lots of same-class and @@ -31,6 +37,151 @@ from tensorflow.python.ops import variable_scope # functions in base.py which should be reused. +_DeferredRestoration = collections.namedtuple( + + "_DeferredRestoration", + [ + # The map_func to use (either user-specified or the default). + "map_func", + # Boolean, True if the user specified an explicit map_func, for error + # messages. + "map_func_is_user", + # A mapping from checkpoint names to initial values of not-yet-created + # variables which should be restored. These values come from parsing a + # checkpoint. + "checkpointed_variables_to_restore", + # A mapping from checkpoint name to variable objects of variables which + # have already been restored, for error checking. + "restored_variables", + # The session to restore with (if in graph mode). + "session", + # Names of the Network where the restore was requested, for error + # messages. + "network_name", + "network_scope_name" + ]) + + +def _default_naming_conflict_error_message( + mapped_name, first_variable, second_variable, + network_name, network_scope_name): + return ( + ("The default checkpoint variable name mapping strategy for Network " + "'%s' resulted in a naming conflict. We attempted to strip off the " + "variable prefix for the Network ('%s'), but this resulted in two " + "variables named '%s' (originally '%s' and '%s'). This should only " + "happen when using variable sharing (i.e. the Network contains Networks " + "or Layers which were first added to another Network, and therefore " + "have that Network's variable prefix). One solution is to pass " + "`map_func=lambda n: n` to Network.save and Network.restore to use " + "fully qualified variable names in the checkpoint, although this will " + "require that the variable prefix of the Network being restored into " + "is also '%s'. You may alternatively write an arbitrary mapping.") + % ( + network_name, network_scope_name, mapped_name, + first_variable._shared_name, + second_variable._shared_name, network_scope_name + )) + + +def _restore_custom_map_func_error_message( + mapped_name, first_variable, second_variable, + network_name, network_scope_name): + return ( + ("The map_func passed to Network.restore for the Network '%s' " + "resulted in two variables named '%s' (originally '%s' and '%s'). Since " + "this is also an error on Network.save, this Network was " + "probably not saved with this map_func. Note that map_func " + "always maps from full variable names to checkpoint names; " + "there is no need to specify an inverse mapping.\n\n" + "Try stripping less from the variable names, or renaming parts " + "of the Network. For reference, variables created by sub-Layers " + "of this Network are prefixed with '%s', but if they are " + "re-used after being added to another Network they will have " + "that Network's full variable prefix instead.") % ( + network_name, mapped_name, + first_variable._shared_name, + second_variable._shared_name, + network_scope_name)) + + +def _make_custom_getter_for_deferred_restorations(): + """Returns a custom getter which searches `deferred_restorations`. + + Returns: A tuple of (_custom_getter, deferred_restorations) + _custom_getter: The getter which should be added to variable_scopes where + variables will be created. + deferred_restorations: A list for _DeferredRestoration objects. Typically + empty when the getter is set, and expanded as deferred restorations are + requested. All new deferred restorations should be appended to the end of + the list, where they will have priority over older deferred restorations. + """ + deferred_restorations = [] + + def _custom_getter(getter, name, shape=None, dtype=None, + initializer=None, + *args, **kwargs): + """A custom getter which processes deferred restorations.""" + # Iterate over restorations, newest first (newer restorations will take + # precedence over older restorations, just like with immediate restorations + # into existing variables). + delayed_restoration = None + found_value = False + value_to_restore = None + for delayed_restoration in reversed( + deferred_restorations): + checkpoint_name = delayed_restoration.map_func(name) + if (checkpoint_name + in delayed_restoration.checkpointed_variables_to_restore): + found_value = True + value_to_restore = ( + delayed_restoration.checkpointed_variables_to_restore[ + checkpoint_name]) + if found_value: + break + # value_to_restore may be False because this variable is not in any + # checkpoint we are restoring, or None because we have explicitly set it to + # None when it was previously fetched. In either case, we don't need to + # set an initializer. + if found_value and value_to_restore is not None: + initializer = value_to_restore + shape = None + variable = getter(name, shape=shape, dtype=dtype, initializer=initializer, + *args, **kwargs) + if found_value and value_to_restore is not None: + # Mark as already restored from this checkpoint. + delayed_restoration.checkpointed_variables_to_restore[ + checkpoint_name] = None + if context.in_graph_mode(): + delayed_restoration.session.run(variable.initializer) + if found_value: + # Error checking should run even if we've already restored a value. + if delayed_restoration.restored_variables.setdefault( + checkpoint_name, variable) is not variable: + # Naming conflict. We've tried to initialize two variables with the + # same value from the checkpoint. + if delayed_restoration.map_func_is_user: + raise ValueError( + _restore_custom_map_func_error_message( + mapped_name=checkpoint_name, + first_variable=delayed_restoration.restored_variables[ + checkpoint_name], + second_variable=variable, + network_name=delayed_restoration.network_name, + network_scope_name=delayed_restoration.network_scope_name)) + else: + raise ValueError( + _default_naming_conflict_error_message( + mapped_name=checkpoint_name, + first_variable=delayed_restoration.restored_variables[ + checkpoint_name], + second_variable=variable, + network_name=delayed_restoration.network_name, + network_scope_name=delayed_restoration.network_scope_name)) + return variable + return _custom_getter, deferred_restorations + + class Network(base.Layer): """Represents the composition of a set of Layers. @@ -41,7 +192,6 @@ class Network(base.Layer): - Convert inputs to __call__ to tensors. - Prevent variables from being created after the first __call__? (Think about restoring from a checkpoint). - - Save & restore """ def __init__(self, name=None): @@ -60,6 +210,8 @@ class Network(base.Layer): self._owned_layers = {} # The scope to use if we end up without a parent. self._default_parent_variable_scope = variable_scope.get_variable_scope() + self._custom_getter, self._deferred_restorations = ( + _make_custom_getter_for_deferred_restorations()) def _init_set_name(self, name): # Anonymous Networks (name=None) defer setting a final name until they are @@ -87,7 +239,8 @@ class Network(base.Layer): avoid_names = None self._name, self._base_name = self._make_unique_name( name_uid_map=name_uid_map, avoid_names=avoid_names) - if self._first_parent is None or self._first_parent() is None: + if self._first_parent is None or (self._first_parent # False = no parent + and self._first_parent() is None): # Save a pointer to the parent Network so that we can later check that the # scope name we get is correct. if not parent_network: @@ -151,26 +304,32 @@ class Network(base.Layer): "of Networks in which they were first created). To set " "options, try `with tf.variable_scope(''):`. If this " "limitation bothers you, please file a feature request.") - for non_network_constituent in self._non_network_sublayers: - if non_network_constituent._scope is None: - if non_network_constituent._first_parent is None: - constituent_first_parent = None - else: - constituent_first_parent = non_network_constituent._first_parent() - if constituent_first_parent: - constituent_first_parent._set_scope() - parent_scope = constituent_first_parent._scope - else: - parent_scope = ( - non_network_constituent._default_parent_variable_scope) - with variable_scope.variable_scope(parent_scope): - # Horrid hack to make Layer variable names which are direct - # sub-layers of Networks conform to the Network variable naming - # conventions. - with variable_scope.variable_scope( - None, use_resource=True, - default_name=non_network_constituent.name) as sub_scope: - non_network_constituent._scope = sub_scope + for non_network_sublayer in self._non_network_sublayers: + self._set_scope_for_nonnetwork_sublayer(non_network_sublayer) + + def _set_scope_for_nonnetwork_sublayer(self, sublayer): + if sublayer._scope is None: + if sublayer._first_parent is None: + constituent_first_parent = None + else: + constituent_first_parent = sublayer._first_parent() + if constituent_first_parent: + constituent_first_parent._set_scope() + parent_scope = constituent_first_parent._scope + else: + self._finalize_name(False) + raise ValueError( + ("The parent of a Layer added to Network %s was garbage collected " + "before the Layer was built. If this limitation bothers you " + "please, file a feature request.") % (self.name,)) + with variable_scope.variable_scope(parent_scope): + # Horrid hack to make Layer variable names which are direct + # sub-layers of Networks conform to the Network variable naming + # conventions. + with variable_scope.variable_scope( + None, use_resource=True, + default_name=sublayer.name) as sub_scope: + sublayer._scope = sub_scope @base.Layer.name.getter def name(self): @@ -327,6 +486,270 @@ class Network(base.Layer): "at https://github.com/tensorflow/tensorflow/issues/new if this is " "important to you") + def _strip_variable_prefix(self, original_variable_name): + """The default map_func for saving or restoring variables. + + Strips the variable prefix for the Network on which save/restore was called, + and leaves other variable names fully qualified in the checkpoint. + + Args: + original_variable_name: The _shared_name of the variable (no :0 + suffix) to map. + Returns: + The checkpoint name of the variable. + """ + scope_name_with_slash = self.scope_name + "/" + if original_variable_name.startswith(scope_name_with_slash): + return original_variable_name[len(scope_name_with_slash):] + else: + return original_variable_name + + def save(self, save_path, global_step=None, map_func=None): + """Save variables from the Network to a checkpoint. + + Args: + save_path: Either a checkpoint prefix or the name of a directory to save + the checkpoint in (in which case the checkpoint will be named based on + the Network name). + global_step: The global step to use when naming the checkpoint. If None + (default), we will first try to get the default global step. If that + fails because no default global step exists, then the checkpoint is + created without a global step suffix. + map_func: A function mapping fully qualified variable names + (e.g. 'my_network_1/dense_1/kernel') to names in the checkpoint. By + default (if `map_func=None`), the variable prefix for the network being + restored (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped + and all other variable names (shared with other Networks) are left + unchanged. + Returns: + The checkpoint prefix for the saved checkpoint, which may be passed to + `Network.restore`. + Raises: + ValueError: If the Network has not yet been called, or if map_func results + in a name collision. + """ + if not self.built: + raise ValueError( + "Attempt to save the Network before it was first called. This means " + "variables have not yet been created, so there is nothing to save.") + self._set_scope() # scope_name should be available to map_funcs + if global_step is None: + global_step = training_util.get_global_step() + if os.path.isdir(save_path): + # If we were passed a directory, default to naming based on the Network + # name. + save_path = os.path.join(save_path, self.name) + user_map_func = map_func + if map_func is None: + map_func = self._strip_variable_prefix + variable_map = {} + for variable in self.variables: + mapped_name = map_func(variable._shared_name) + if variable_map.setdefault(mapped_name, variable) is not variable: + if user_map_func is None: + # Instead of erroring out, we could just re-try and silently use the + # full variable names in the checkpoint. This could be odd for deeply + # nested sub-Networks (since the full prefix from the nesting would + # get added), so for now we'll let the user deal with this case. + raise ValueError(_default_naming_conflict_error_message( + mapped_name=mapped_name, + first_variable=variable_map[mapped_name], + second_variable=variable, + network_name=self.name, + network_scope_name=self.scope_name)) + else: + # The user passed their own problematic map_func. + raise ValueError( + ("The map_func passed to Network.save for the Network '%s' " + "resulted in two variables named '%s' ('%s' and '%s'). Try " + "stripping less from the variable names, or renaming parts of " + "the Network. For reference, variables created by sub-Layers of " + "this Network are prefixed with '%s', but if they are re-used " + "after being added to another Network, they will have that " + "Network's full variable prefix instead.") % ( + self.name, mapped_name, + variable_map[mapped_name]._shared_name, + variable._shared_name, + self.scope_name)) + if context.in_eager_mode(): + sess = None + else: + sess = ops.get_default_session() + return saver_lib.Saver(variable_map).save( + sess=sess, save_path=save_path, write_meta_graph=False, + global_step=global_step) + + def _restore_existing_variables(self, save_path, map_func, user_map_func): + """Use a standard Saver to restore existing variables from a checkpoint. + + Args: + save_path: The checkpoint prefix or directory to read from. + map_func: The function to use when mapping from variable names to + checkpoint names. + user_map_func: The original map_func passed by the user, for error + checking. + Returns: + A dictionary mapping from checkpoint names to variable objects which have + been restored (for bookkeeping to avoid deferred restorations on these + variables). + Raises: + ValueError: If there is a name collision. + """ + existing_variables_by_checkpoint_name = {} + for variable in self.variables: + checkpoint_name = map_func(variable._shared_name) + if existing_variables_by_checkpoint_name.setdefault( + checkpoint_name, variable) is not variable: + if user_map_func is None: + raise ValueError(_default_naming_conflict_error_message( + mapped_name=checkpoint_name, + first_variable=existing_variables_by_checkpoint_name[ + checkpoint_name], + second_variable=variable, + network_name=self.name, + network_scope_name=self.scope_name)) + else: + raise ValueError(_restore_custom_map_func_error_message( + mapped_name=checkpoint_name, + first_variable=existing_variables_by_checkpoint_name[ + checkpoint_name], + second_variable=variable, + network_name=self.name, + network_scope_name=self.scope_name)) + if existing_variables_by_checkpoint_name: + if context.in_eager_mode(): + sess = None + else: + sess = ops.get_default_session() + saver_lib.Saver(var_list=existing_variables_by_checkpoint_name).restore( + sess=sess, save_path=save_path) + return existing_variables_by_checkpoint_name + + def _set_restore_on_create(self, save_path, map_func, user_map_func, + existing_variables_by_checkpoint_name): + """If necessary, request deferred restorations of variables.""" + checkpoint_reader = checkpoint_utils.load_checkpoint(save_path) + checkpointed_variables_to_restore = {} + for checkpoint_name, _ in checkpoint_utils.list_variables(save_path): + if checkpoint_name in existing_variables_by_checkpoint_name: + # This variable was already created and restored. + continue + # Save the variable for later restoration in a custom getter. + checkpointed_variables_to_restore[checkpoint_name] = ( + checkpoint_reader.get_tensor(checkpoint_name)) + # Only set a deferred restoration if there are checkpoint variables which + # have not been assigned to existing variables. Note that this loses out on + # some opportunity for error checking, but avoids creating + # _DeferredRestoration objects once a Network has been built (so that + # restoring in a loop does not take increasing amounts of memory). + if checkpointed_variables_to_restore: + if context.in_eager_mode(): + sess = None + else: + sess = ops.get_default_session() + # We need a name for error messages. If we haven't been added to another + # Network yet, we're top-level. + self._finalize_name(False) + self._set_scope() + # Save a record of this restoration for use in the custom getter. + deferred_restoration = _DeferredRestoration( + map_func=map_func, + map_func_is_user=(user_map_func is not None), + checkpointed_variables_to_restore=checkpointed_variables_to_restore, + restored_variables={}, + session=sess, + network_name=self.name, + network_scope_name=self.scope_name) + self._deferred_restorations.append(deferred_restoration) + # Add the deferred registration to non-Network children, and request that + # Networks propagate the request to their children. + self._add_deferred_restoration(deferred_restoration) + + def _add_deferred_restoration(self, deferred_restoration): + """Add a deferred restoration to this Network and all children. + + Restorations which are requested later have higher priority, and the highest + priority matching restoration is applied to a variable when it is created. + + Args: + deferred_restoration: A _DeferredRestoration object. + """ + # Networks don't create variables at the moment, so this append isn't + # strictly necessary. We could get by with only adding deferred restorations + # to non-Network Layers. + self._set_scope() + # We use set_custom_getter because it avoids recursively calling up the + # variable_scope tree. We've done the tree traversal ourselves and have + # added the request to each Layer which needs it. + self._scope.set_custom_getter(self._custom_getter) + self._deferred_restorations.append(deferred_restoration) + for layer in self.layers: + if isinstance(layer, Network): + # For Networks, request that they propagate this deferred restoration + # to all of their children recursively. + layer._add_deferred_restoration(deferred_restoration) + else: + # For non-Network Layers, make sure they have a deferred restoration + # queue and a custom getter, then add our request to it. + if not hasattr(layer, "_custom_getter"): + assert not hasattr(layer, "_deferred_restorations") + layer._custom_getter, layer._deferred_restorations = ( + _make_custom_getter_for_deferred_restorations()) + self._set_scope_for_nonnetwork_sublayer(layer) + layer._scope.set_custom_getter(layer._custom_getter) + layer._deferred_restorations.append(deferred_restoration) + + def restore(self, save_path, map_func=None): + """Restore the Network from a checkpoint. + + If variables have already been created (typically when some or all of the + `Network` is built), they are assigned values from the checkpoint + immediately, overwriting any existing values (in graph mode the default + session is used for the assignments). + + If there are checkpoint entries which do not correspond to any existing + variables in the `Network`, these values are saved for deferred restoration; + their initial values will be the checkpointed values once they are + created. Requests for multiple deferred restorations behave the same way as + immediate restorations, in that later requests will take priority over + earlier requests relevant to the same variable. + + If this `Network` shares `Layer`s with another network, those `Layer`s will + also have their variables restored from the checkpoint. + + Args: + save_path: The return value of `Network.save`, or a directory to search + for a checkpoint. + map_func: A function mapping fully qualified variable names + (e.g. 'my_network_1/dense_1/kernel') to names in the checkpoint. By + default (if `map_func=None`), the variable prefix for the network being + restored (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped + and all other variable names (shared with other Networks) are left + unchanged. Note that this is the _same_ map_func as `Network.save`, not + an inverse mapping. + """ + self._finalize_name(parent_network=False) + self._set_scope() # scope_name should be available to map_funcs + if os.path.isdir(save_path): + # If we don't have a name yet, set no parent. + save_path = os.path.join(save_path, self.name) + user_map_func = map_func + if map_func is None: + map_func = self._strip_variable_prefix + # Step one is to restore any existing variables from the checkpoint. + existing_variables_by_checkpoint_name = self._restore_existing_variables( + save_path=save_path, + map_func=map_func, + user_map_func=user_map_func) + # Step two is to set a custom getter which restores variables on creation, + # for those variables which have not been added to sub-Layers yet. + self._set_restore_on_create( + save_path=save_path, + map_func=map_func, + user_map_func=user_map_func, + existing_variables_by_checkpoint_name=( + existing_variables_by_checkpoint_name)) + # TODO(josh11b): Support other Layer methods needed for graph mode, such as for # losses and updates diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py index e4cba3f2ed..c621f527c2 100644 --- a/tensorflow/contrib/eager/python/network_test.py +++ b/tensorflow/contrib/eager/python/network_test.py @@ -21,12 +21,14 @@ import gc from tensorflow.contrib.eager.python import network from tensorflow.python.eager import test from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import test_util from tensorflow.python.layers import core from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope +from tensorflow.python.training import training_util # pylint: disable=not-callable @@ -42,6 +44,29 @@ class MyNetwork(network.Network): class NetworkTest(test.TestCase): + def _save_modify_load_network_built(self, net, global_step=None): + checkpoint_directory = self.get_temp_dir() + checkpoint_path = net.save( + save_path=checkpoint_directory, global_step=global_step) + input_value = constant_op.constant([[42.0]]) + original_output = self.evaluate(net(input_value)) + for var in net.variables: + self.evaluate(var.assign(var + 1.)) + self.assertGreater( + self.evaluate(net(input_value)), + original_output) + # Either the returned explicit checkpoint path or the directory should work. + net.restore(save_path=checkpoint_directory) + self.assertAllEqual( + original_output, + self.evaluate(net(input_value))) + for var in net.variables: + self.evaluate(var.assign(var + 2.)) + net.restore(save_path=checkpoint_path) + self.assertAllEqual( + original_output, + self.evaluate(net(input_value))) + @test_util.run_in_graph_and_eager_modes() def testTrainableAttribute(self): net = network.Network() @@ -60,6 +85,264 @@ class NetworkTest(test.TestCase): result = net(constant_op.constant([[2.0]])) self.assertEqual(34.0, self.evaluate(result)) + @test_util.run_in_graph_and_eager_modes() + def testNetworkSaveRestoreAlreadyBuilt(self): + net = MyNetwork(name="abcd") + with self.assertRaisesRegexp( + ValueError, "Attempt to save the Network before it was first called"): + net.save(self.get_temp_dir()) + net(constant_op.constant([[2.0]])) + self.evaluate(net.trainable_variables[0].assign([[17.0]])) + self._save_modify_load_network_built(net, global_step=None) + self._save_modify_load_network_built(net, global_step=10) + + @test_util.run_in_graph_and_eager_modes() + def testSaveRestoreDefaultGlobalStep(self): + net = MyNetwork(name="abcd") + net(constant_op.constant([[2.0]])) + self.evaluate(net.variables[0].assign([[3.]])) + default_global_step = training_util.get_or_create_global_step() + self.evaluate(default_global_step.assign(4242)) + save_path = net.save(self.get_temp_dir()) + self.assertIn("abcd-4242", save_path) + + @test_util.run_in_graph_and_eager_modes() + def testNetworkSaveAndRestoreIntoUnbuilt(self): + save_dir = self.get_temp_dir() + net1 = MyNetwork() + test_input = constant_op.constant([[2.0]]) + net1(test_input) + self.evaluate(net1.trainable_variables[0].assign([[17.0]])) + save_path = net1.save(save_dir) + # With a pre-build restore we should have the same value. + net2 = MyNetwork() + net2.restore(save_path) + self.assertAllEqual(self.evaluate(net1(test_input)), + self.evaluate(net2(test_input))) + self.assertIsNot(net1.variables[0], net2.variables[0]) + self.assertAllEqual(self.evaluate(net1.variables[0]), + self.evaluate(net2.variables[0])) + + @test_util.run_in_graph_and_eager_modes() + def testLoadIntoUnbuiltSharedLayer(self): + + class Owner(network.Network): + + def __init__(self, name=None): + super(Owner, self).__init__(name=name) + self.first = self.track_layer(core.Dense( + 1, name="first_layer", use_bias=False)) + + def call(self, x): + return self.first(x) + + first_owner = Owner() + + class User(network.Network): + + def __init__(self, use_layer, name=None): + super(User, self).__init__(name=name) + self.first = self.track_layer(use_layer) + self.second = self.track_layer(core.Dense( + 1, name="second_layer", use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + class LikeUserButNotSharing(network.Network): + + def __init__(self, name=None): + super(LikeUserButNotSharing, self).__init__(name=name) + self.first = self.track_layer(core.Dense( + 1, name="first_layer", use_bias=False)) + self.second = self.track_layer(core.Dense( + 1, name="second_layer", use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + checkpoint_creator = LikeUserButNotSharing(name="checkpoint_creator") + one = constant_op.constant([[1.0]]) + checkpoint_creator(one) + self.assertEqual(2, len(checkpoint_creator.variables)) + self.evaluate(checkpoint_creator.variables[0].assign([[5.]])) + self.evaluate(checkpoint_creator.variables[1].assign([[6.]])) + # Re-map the variable names so that with default restore mapping we'll + # attempt to restore into the unbuilt Layer. + name_mapping = { + "checkpoint_creator/first_layer/kernel": "owner_1/first_layer/kernel", + "checkpoint_creator/second_layer/kernel": "second_layer/kernel", + } + save_path = checkpoint_creator.save( + self.get_temp_dir(), + map_func=lambda full_name: name_mapping[full_name]) + load_into = User(use_layer=first_owner.first) + load_into.restore(save_path) + self.assertEqual(0, len(first_owner.variables)) + self.assertAllEqual(self.evaluate(checkpoint_creator(one)), + self.evaluate(load_into(one))) + self.assertEqual(1, len(first_owner.variables)) + self.assertAllEqual([[5.]], self.evaluate(load_into.variables[0])) + self.assertAllEqual([[6.]], self.evaluate(load_into.variables[1])) + first_owner(one) + self.assertAllEqual([[5.]], self.evaluate(first_owner.variables[0])) + + # Try again with a garbage collected parent. + first_owner = Owner() + load_into = User(use_layer=first_owner.first) + del first_owner + gc.collect() + def _restore_map_func(original_name): + if original_name.startswith("owner_1"): + return original_name.replace("owner_1", "owner_2") + else: + return "user_2/" + original_name + with self.assertRaisesRegexp(ValueError, "garbage collected"): + load_into.restore(save_path, map_func=_restore_map_func) + + @test_util.run_in_graph_and_eager_modes() + def testRestoreIntoSubNetwork(self): + + class Parent(network.Network): + + def __init__(self, name=None): + super(Parent, self).__init__(name=name) + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.first(self.second(x)) + + one = constant_op.constant([[3.]]) + whole_model_saver = Parent() + whole_model_saver(one) + self.evaluate(whole_model_saver.variables[0].assign([[15.]])) + self.evaluate(whole_model_saver.variables[1].assign([[16.]])) + whole_model_checkpoint = whole_model_saver.save(self.get_temp_dir()) + + save_from = MyNetwork() + save_from(one) + self.evaluate(save_from.variables[0].assign([[5.]])) + checkpoint = save_from.save(self.get_temp_dir()) + save_into_parent = Parent() + save_into_parent.restore(whole_model_checkpoint) + save_into_parent.first.restore(checkpoint) + save_into_parent.first.restore(checkpoint) # deferred loading multiple + # times is fine + save_into_parent(one) # deferred loading + self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[0])) + self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1])) + + # Try again with the opposite ordering, and we should get different results + # (deferred restoration should happen the same way non-deferred happens, + # with later restorations overwriting older ones). + save_into_parent = Parent() + save_into_parent.first.restore(checkpoint) # deferred loading multiple + # times is fine + save_into_parent.restore(whole_model_checkpoint) + save_into_parent(one) # deferred loading + # We've overwritten the sub-Network restore. + self.assertAllEqual([[15.]], self.evaluate(save_into_parent.variables[0])) + self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1])) + + self.evaluate(save_into_parent.variables[0].assign([[3.]])) + self.evaluate(save_into_parent.variables[1].assign([[4.]])) + save_into_parent.second.restore(checkpoint) + self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[1])) + with self.assertRaisesRegexp(errors_impl.NotFoundError, + "not found in checkpoint"): + # The checkpoint is incompatible. + save_into_parent.restore(checkpoint) + + @test_util.run_in_graph_and_eager_modes() + def testCustomMapCollisionErrors(self): + + class Parent(network.Network): + + def __init__(self, name=None): + super(Parent, self).__init__(name=name) + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.first(self.second(x)) + + make_checkpoint = Parent() + one = constant_op.constant([[1.]]) + make_checkpoint(one) + self.evaluate(make_checkpoint.variables[0].assign([[2.]])) + self.evaluate(make_checkpoint.variables[1].assign([[3.]])) + with self.assertRaisesRegexp( + ValueError, + "The map_func passed to Network.save for the Network 'parent_1' " + "resulted in two variables named 'foo'"): + make_checkpoint.save(self.get_temp_dir(), map_func=lambda n: "foo") + checkpoint = make_checkpoint.first.save( + self.get_temp_dir(), map_func=lambda n: "foo") + loader = Parent() + loader.restore(checkpoint, map_func=lambda n: "foo") + with self.assertRaisesRegexp( + ValueError, + ("The map_func passed to Network.restore for the Network" + " 'parent_2' resulted in two variables named 'foo'")): + loader(one) + loader = Parent() + loader(one) + with self.assertRaisesRegexp( + ValueError, + ("The map_func passed to Network.restore for the Network" + " 'parent_3' resulted in two variables named 'foo'")): + loader.restore(checkpoint, map_func=lambda n: "foo") + + @test_util.run_in_graph_and_eager_modes() + def testDefaultMapCollisionErrors(self): + + one = constant_op.constant([[1.]]) + first = core.Dense(1, name="dense_1", use_bias=False) + first(one) + + class Parent(network.Network): + + def __init__(self, name=None): + super(Parent, self).__init__(name=name) + self.first = self.track_layer(first) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.first(self.second(x)) + + make_checkpoint = Parent() + one = constant_op.constant([[1.]]) + make_checkpoint(one) + self.evaluate(make_checkpoint.variables[0].assign([[2.]])) + self.evaluate(make_checkpoint.variables[1].assign([[3.]])) + with self.assertRaisesRegexp( + ValueError, + ("The default checkpoint variable name mapping strategy for Network " + "'parent_1' resulted in a naming conflict.")): + make_checkpoint.save(self.get_temp_dir()) + + class Compatible(network.Network): + + def __init__(self, name=None): + super(Compatible, self).__init__(name=name) + self.first = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.first(x) + + successful_checkpoint = Compatible() + successful_checkpoint(one) + self.evaluate(successful_checkpoint.variables[0].assign([[-1.]])) + checkpoint_path = successful_checkpoint.save(self.get_temp_dir()) + load_checkpoint = Parent() + load_checkpoint(one) + with self.assertRaisesRegexp( + ValueError, + ("The default checkpoint variable name mapping strategy for Network " + "'parent_2' resulted in a naming conflict.")): + load_checkpoint.restore(checkpoint_path) + def testNoReferenceCyclesAfterCall(self): class ChildNetwork(network.Network): -- GitLab From 7c4e98eb4a459eaf79fb76a97c35481b8f063c85 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Fri, 27 Oct 2017 12:20:47 -0700 Subject: [PATCH 472/573] Add Tensor._rank() getter It appears to speed up SPINN model by about 1%, which is not much, but this method is very simple and easier to use than len(tensor._shape_tuple()) PiperOrigin-RevId: 173703259 --- tensorflow/python/eager/pywrap_tensor.cc | 10 ++++++++++ tensorflow/python/framework/ops.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc index 157e87d387..3adaea2c79 100644 --- a/tensorflow/python/eager/pywrap_tensor.cc +++ b/tensorflow/python/eager/pywrap_tensor.cc @@ -377,6 +377,15 @@ static PyObject* EagerTensor_shape_tuple(EagerTensor* self) { return shape; } +// Getter for `_rank`. +static PyObject* EagerTensor_rank(EagerTensor* self) { +#if PY_MAJOR_VERSION < 3 + return PyInt_FromLong(TFE_TensorHandleNumDims(self->handle)); +#else + return PyLong_FromLong(TFE_TensorHandleNumDims(self->handle)); +#endif +} + static PyObject* EagerTensor_tensor_handle(EagerTensor* self, void* unused) { Py_INCREF(self->handle_data); return self->handle_data; @@ -470,6 +479,7 @@ static PyMethodDef EagerTensor_methods[] = { PyDoc_STR("_datatype_enum")}, {"_shape_tuple", (PyCFunction)EagerTensor_shape_tuple, METH_NOARGS, PyDoc_STR("_shape_tuple")}, + {"_rank", (PyCFunction)EagerTensor_rank, METH_NOARGS, PyDoc_STR("_rank")}, {"_copy_to_device", (PyCFunction)EagerTensor_copy_to_device, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("_copy_to_device")}, {nullptr, nullptr}, diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index c278fb2a39..c8ee9243d7 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -383,6 +383,14 @@ class Tensor(_TensorLike): return None return tuple(shape) + def _rank(self): + """Integer rank of this Tensor, if known, else None. + + Returns: + Integer rank or None + """ + return self._shape.ndims + def get_shape(self): """Alias of Tensor.shape.""" return self.shape @@ -664,6 +672,18 @@ class _EagerTensorBase(Tensor): """ raise NotImplementedError() + def _rank(self): + """Integer rank of this Tensor. + + Unlike regular Tensors, the rank is always known for EagerTensors. + + This is more performant than len(self._shape_tuple()) + + Returns: + Integer rank + """ + raise NotImplementedError() + def _copy_to_device(self, context, device): # pylint: disable=redefined-outer-name raise NotImplementedError() -- GitLab From 466b9ecf8b42bd4a596281f1b430f122c5c3e00a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 13:01:09 -0700 Subject: [PATCH 473/573] Report total number of bytes to be transferred when the curl request makes no progress. PiperOrigin-RevId: 173707608 --- tensorflow/core/platform/cloud/curl_http_request.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/platform/cloud/curl_http_request.cc b/tensorflow/core/platform/cloud/curl_http_request.cc index e1f8867b38..e2d935f35e 100644 --- a/tensorflow/core/platform/cloud/curl_http_request.cc +++ b/tensorflow/core/platform/cloud/curl_http_request.cc @@ -512,8 +512,10 @@ int CurlHttpRequest::ProgressCallback(void* this_object, curl_off_t dltotal, } if (now - that->last_progress_timestamp_ > kInactivityTimeoutSeconds) { - LOG(ERROR) << "The transmission has been stuck at " << current_progress - << " bytes for " << now - that->last_progress_timestamp_ + LOG(ERROR) << "The transmission of request " << this_object + << " has been stuck at " << current_progress << " of " + << dltotal + ultotal << " bytes for " + << now - that->last_progress_timestamp_ << " seconds and will be aborted."; return 1; // Will abort the request. } -- GitLab From b31b08bb0f876db76ae3beb0b0801ab1893f9abf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 13:14:41 -0700 Subject: [PATCH 474/573] Adds randomized tests for newly introduced complex and related ops. PiperOrigin-RevId: 173709206 --- tensorflow/compiler/tests/randomized_tests.cc | 236 +++++++++++------- .../compiler/tf2xla/kernels/unary_ops.cc | 2 +- 2 files changed, 145 insertions(+), 93 deletions(-) diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc index 461af83362..c8a32f9e29 100644 --- a/tensorflow/compiler/tests/randomized_tests.cc +++ b/tensorflow/compiler/tests/randomized_tests.cc @@ -367,11 +367,11 @@ OpTest::OpTest() { void OpTest::Repeatedly(const std::function& fn) { int const max_repetitions = tf_xla_test_repetitions; int valid_test_runs = 0; - // We run up to 20 * max_repetitions times; the idea is that if we roll the + // We run up to 100 * max_repetitions times; the idea is that if we roll the // dice enough times we will find some valid parameters. We want to put an // upper limit on the number iterations just in case the probability of // finding feasible parameters is very low. - for (int i = 0; !HasFailure() && i < max_repetitions * 20 && + for (int i = 0; !HasFailure() && i < max_repetitions * 100 && valid_test_runs < max_repetitions; ++i) { TestResult result = fn(); @@ -868,7 +868,7 @@ Tensor AsIntTensor(DataType dtype, const std::vector& values) { TEST_F(OpTest, Abs) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Abs").RandomInput(type).Attr("T", type)); }); @@ -883,7 +883,7 @@ TEST_F(OpTest, Acosh) { TEST_F(OpTest, Add) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Add") .RandomInput(type, dims.first) @@ -894,7 +894,7 @@ TEST_F(OpTest, Add) { TEST_F(OpTest, AddN) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); int n = std::uniform_int_distribution(1, 5)(generator()); auto shape = RandomDims(); @@ -921,6 +921,14 @@ TEST_F(OpTest, All) { }); } +TEST_F(OpTest, Angle) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Angle") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, Any) { Repeatedly([this]() { std::vector data_dims = RandomDims(); @@ -935,11 +943,11 @@ TEST_F(OpTest, Any) { TEST_F(OpTest, ApproximateEqual) { Repeatedly([this]() { - auto dims = RandomDims(); - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto dims = BroadcastableDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ApproximateEqual") - .RandomInput(type, dims) - .RandomInput(type, dims) + .RandomInput(type, dims.first) + .RandomInput(type, dims.second) .Attr("T", DT_FLOAT)); }); } @@ -990,6 +998,16 @@ TEST_F(OpTest, Atanh) { }); } +TEST_F(OpTest, Atan2) { + Repeatedly([this]() { + auto dims = BroadcastableDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Atan2") + .RandomInput(DT_FLOAT, dims.first) + .RandomInput(DT_FLOAT, dims.second) + .Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, AvgPool) { Repeatedly([this]() { std::uniform_int_distribution random_int(1, 5); @@ -1085,7 +1103,7 @@ TEST_F(OpTest, AvgPool3DGrad) { TEST_F(OpTest, BatchMatMul) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); std::vector output_dims = RandomDims(2, 5, 0, 7); int64 ndims = output_dims.size(); int64 inner_dim = RandomDim(); @@ -1138,7 +1156,7 @@ TEST_F(OpTest, BatchToSpace) { CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals), TensorShape({num_block_dims, 2}))); - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchToSpace") .RandomInput(type, input_dims) .Input(crops) @@ -1176,7 +1194,7 @@ TEST_F(OpTest, BatchToSpaceND) { CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals), TensorShape({num_block_dims, 2}))); - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("BatchToSpaceND") .RandomInput(type, input_dims) @@ -1192,7 +1210,7 @@ TEST_F(OpTest, BiasAdd) { auto x_dims = RandomDims(2, kDefaultMaxRank); auto y_dims = {x_dims[x_dims.size() - 1]}; // TODO(phawkins): test both data formats. - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAdd") .RandomInput(type, x_dims) .RandomInput(type, y_dims) @@ -1203,7 +1221,7 @@ TEST_F(OpTest, BiasAdd) { TEST_F(OpTest, BiasAddGrad) { Repeatedly([this]() { // TODO(phawkins): test both data formats. - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("BiasAddGrad").RandomInput(type).Attr("T", type)); }); @@ -1213,7 +1231,7 @@ TEST_F(OpTest, BiasAddV1) { Repeatedly([this]() { auto x_dims = RandomDims(2, kDefaultMaxRank); auto y_dims = {x_dims[x_dims.size() - 1]}; - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAddV1") .RandomInput(type, x_dims) .RandomInput(type, y_dims) @@ -1246,7 +1264,7 @@ TEST_F(OpTest, BitwiseOr) { TEST_F(OpTest, BroadcastArgs) { Repeatedly([this]() { // TODO(phawkins): only int32 seems to be implemented in Tensorflow. - // DataType type = Choose({DT_INT32, DT_INT64}); + // auto type = Choose({DT_INT32, DT_INT64}); DataType type = DT_INT32; auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose( @@ -1260,7 +1278,7 @@ TEST_F(OpTest, BroadcastArgs) { TEST_F(OpTest, BroadcastGradientArgs) { Repeatedly([this]() { // TODO(phawkins): only int32 seems to be implemented in Tensorflow. - // DataType type = Choose({DT_INT32, DT_INT64}); + // auto type = Choose({DT_INT32, DT_INT64}); DataType type = DT_INT32; auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose( @@ -1290,9 +1308,19 @@ TEST_F(OpTest, Ceil) { }); } +TEST_F(OpTest, Complex) { + Repeatedly([this]() { + auto dims = BroadcastableDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Complex") + .RandomInput(DT_FLOAT, dims.first) + .RandomInput(DT_FLOAT, dims.second) + .Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, Concat) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(2, 5)(generator()); std::vector dims = RandomDims(1); @@ -1332,6 +1360,14 @@ TEST_F(OpTest, ConcatOffset) { }); } +TEST_F(OpTest, Conj) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Conj") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, Conv2D) { Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); @@ -1471,7 +1507,7 @@ TEST_F(OpTest, Conv3DBackpropInput) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out}; - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3DBackpropInputV2") .Input(in_shape) @@ -1485,7 +1521,7 @@ TEST_F(OpTest, Conv3DBackpropInput) { TEST_F(OpTest, Cos) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Cos").RandomInput(type).Attr("T", type)); }); @@ -1493,7 +1529,7 @@ TEST_F(OpTest, Cos) { TEST_F(OpTest, Cosh) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Cosh").RandomInput(type).Attr("T", type)); }); @@ -1506,7 +1542,7 @@ TEST_F(OpTest, DepthToSpace) { input_dims[1] = (input_dims[1] + (block - 1)) / block; input_dims[2] = (input_dims[2] + (block - 1)) / block; input_dims[3] *= block * block; - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("DepthToSpace") .RandomInput(type, input_dims) .Attr("T", type) @@ -1597,7 +1633,7 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) { TEST_F(OpTest, Diag) { if (1) return; Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims; // Diag causes a quadratic blowup in output size. int64 size; @@ -1612,7 +1648,7 @@ TEST_F(OpTest, Diag) { TEST_F(OpTest, DiagPart) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); auto dims = RandomDims(1, 3); // Duplicate the random dims. std::vector doubled_dims(dims.size() * 2); @@ -1626,7 +1662,7 @@ TEST_F(OpTest, DiagPart) { TEST_F(OpTest, Div) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Div") .RandomInput(type, dims.first) @@ -1637,7 +1673,7 @@ TEST_F(OpTest, Div) { TEST_F(OpTest, DynamicStitch) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(2, 5)(generator()); OpTestBuilder builder("DynamicStitch"); builder.Attr("T", type); @@ -1722,7 +1758,7 @@ TEST_F(OpTest, SeluGrad) { TEST_F(OpTest, Equal) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Equal") .RandomInput(type, dims.first) @@ -1733,7 +1769,7 @@ TEST_F(OpTest, Equal) { TEST_F(OpTest, Exp) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Exp").RandomInput(type).Attr("T", type)); }); @@ -1741,7 +1777,7 @@ TEST_F(OpTest, Exp) { TEST_F(OpTest, Expm1) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Expm1").RandomInput(type).Attr("T", type)); }); @@ -1749,7 +1785,7 @@ TEST_F(OpTest, Expm1) { TEST_F(OpTest, ExpandDims) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector in_dims = RandomDims(); Tensor dim(DT_INT32, TensorShape()); std::uniform_int_distribution d(-1 - in_dims.size(), in_dims.size()); @@ -1763,7 +1799,7 @@ TEST_F(OpTest, ExpandDims) { TEST_F(OpTest, Fill) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(); std::vector shape(dims.begin(), dims.end()); return ExpectTfAndXlaOutputsAreClose( @@ -1794,7 +1830,7 @@ TEST_F(OpTest, FloorDiv) { TEST_F(OpTest, FloorMod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("FloorMod") .RandomInput(type, dims.first) @@ -1805,7 +1841,7 @@ TEST_F(OpTest, FloorMod) { TEST_F(OpTest, Greater) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Greater") .RandomInput(type, dims.first) @@ -1816,7 +1852,7 @@ TEST_F(OpTest, Greater) { TEST_F(OpTest, GreaterEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("GreaterEqual") .RandomInput(type, dims.first) @@ -1825,6 +1861,14 @@ TEST_F(OpTest, GreaterEqual) { }); } +TEST_F(OpTest, Imag) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Imag") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, Invert) { Repeatedly([this]() { DataType type = DT_INT32; @@ -1843,7 +1887,7 @@ TEST_F(OpTest, L2Loss) { TEST_F(OpTest, Less) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Less") .RandomInput(type, dims.first) @@ -1854,7 +1898,7 @@ TEST_F(OpTest, Less) { TEST_F(OpTest, LessEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("LessEqual") .RandomInput(type, dims.first) @@ -1870,7 +1914,7 @@ TEST_F(OpTest, LinSpace) { return test::AsScalar(x); }; std::uniform_int_distribution distribution(-50, 50); - DataType type = Choose({DT_INT32, DT_INT64}); + auto type = Choose({DT_INT32, DT_INT64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("LinSpace") .RandomInput(DT_FLOAT, {}) @@ -1883,7 +1927,7 @@ TEST_F(OpTest, LinSpace) { TEST_F(OpTest, Log) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Log").RandomInput(type).Attr("T", type)); }); @@ -1891,7 +1935,7 @@ TEST_F(OpTest, Log) { TEST_F(OpTest, Log1p) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Log1p").RandomInput(type).Attr("T", DT_FLOAT)); }); @@ -1990,7 +2034,7 @@ TEST_F(OpTest, MatMul) { std::swap(b_dims[0], b_dims[1]); } - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatMul") .RandomInput(type, a_dims) .RandomInput(type, b_dims) @@ -2002,7 +2046,7 @@ TEST_F(OpTest, MatMul) { TEST_F(OpTest, MatrixDiag) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiag") .RandomInput(type, RandomDims(1)) .Attr("T", type)); @@ -2011,7 +2055,7 @@ TEST_F(OpTest, MatrixDiag) { TEST_F(OpTest, MatrixDiagPart) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiagPart") .RandomInput(type, RandomDims(2)) .Attr("T", type)); @@ -2020,7 +2064,7 @@ TEST_F(OpTest, MatrixDiagPart) { TEST_F(OpTest, Max) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2034,7 +2078,7 @@ TEST_F(OpTest, Max) { TEST_F(OpTest, Maximum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Maximum") .RandomInput(type, dims.first) @@ -2102,7 +2146,7 @@ TEST_F(OpTest, MaxPool3D) { TEST_F(OpTest, Mean) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); // TODO(phawkins): CPU and XLA differ output for reducing across a // size-0 dimension (nan vs 0). For now, require size >= 1. std::vector data_dims = RandomDims(0, kDefaultMaxRank, 1); @@ -2118,7 +2162,7 @@ TEST_F(OpTest, Mean) { TEST_F(OpTest, Min) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2132,7 +2176,7 @@ TEST_F(OpTest, Min) { TEST_F(OpTest, Minimum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Minimum") .RandomInput(type, dims.first) @@ -2153,7 +2197,7 @@ TEST_F(OpTest, Mod) { TEST_F(OpTest, Mul) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Mul") .RandomInput(type, dims.first) @@ -2164,7 +2208,7 @@ TEST_F(OpTest, Mul) { TEST_F(OpTest, Neg) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Neg").RandomInput(type).Attr("T", type)); }); @@ -2172,7 +2216,7 @@ TEST_F(OpTest, Neg) { TEST_F(OpTest, NotEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("NotEqual") .RandomInput(type, dims.first) @@ -2183,7 +2227,7 @@ TEST_F(OpTest, NotEqual) { TEST_F(OpTest, OneHot) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(); int num_dims = dims.size(); @@ -2213,7 +2257,7 @@ TEST_F(OpTest, OneHot) { TEST_F(OpTest, OnesLike) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("OnesLike").RandomInput(type).Attr("T", type)); }); @@ -2221,7 +2265,7 @@ TEST_F(OpTest, OnesLike) { TEST_F(OpTest, Pack) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(1, 5)(generator()); std::vector dims = RandomDims(); @@ -2243,7 +2287,7 @@ TEST_F(OpTest, Pack) { // TODO(b/31741898): crashes on GPU. TEST_F(OpTest, Pad) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector t_dims = RandomDims(); // TODO(b/31741996): re-enable DT_INT64 when bug is fixed. @@ -2272,7 +2316,7 @@ TEST_F(OpTest, Pow) { // nontermination. Repeatedly([this]() { auto dims = BroadcastableDims(); - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Pow") .RandomInput(type, dims.first) .RandomInput(type, dims.second) @@ -2282,7 +2326,7 @@ TEST_F(OpTest, Pow) { TEST_F(OpTest, Prod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2316,15 +2360,23 @@ TEST_F(OpTest, Range) { TEST_F(OpTest, Rank) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Rank").RandomInput(type).Attr("T", type)); }); } +TEST_F(OpTest, Real) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Real") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, RealDiv) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RealDiv") .RandomInput(type, dims.first) @@ -2335,7 +2387,7 @@ TEST_F(OpTest, RealDiv) { TEST_F(OpTest, Reciprocal) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Reciprocal").RandomInput(type).Attr("T", type)); }); @@ -2344,7 +2396,7 @@ TEST_F(OpTest, Reciprocal) { TEST_F(OpTest, ReciprocalGrad) { Repeatedly([this]() { std::vector dims = RandomDims(); - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReciprocalGrad") .RandomInput(type, dims) .RandomInput(type, dims) @@ -2387,7 +2439,7 @@ TEST_F(OpTest, ReluGrad) { TEST_F(OpTest, Reshape) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(); std::bernoulli_distribution random_bool; std::vector dims_before, dims_after; @@ -2415,7 +2467,7 @@ TEST_F(OpTest, Reshape) { TEST_F(OpTest, Reverse) { Repeatedly([this]() { std::vector dims = RandomDims(1); - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int64 rank = dims.size(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Reverse") .RandomInput(type, dims) @@ -2426,7 +2478,7 @@ TEST_F(OpTest, Reverse) { TEST_F(OpTest, ReverseV2) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReverseV2") @@ -2452,7 +2504,7 @@ TEST_F(OpTest, Round) { TEST_F(OpTest, Rsqrt) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Rsqrt").RandomInput(type).Attr("T", type)); }); @@ -2461,7 +2513,7 @@ TEST_F(OpTest, Rsqrt) { TEST_F(OpTest, RsqrtGrad) { Repeatedly([this]() { auto dims = RandomDims(); - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RsqrtGrad") .RandomInput(type, dims) .RandomInput(type, dims) @@ -2471,7 +2523,7 @@ TEST_F(OpTest, RsqrtGrad) { TEST_F(OpTest, Shape) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Shape").RandomInput(type).Attr("T", type)); }); @@ -2479,7 +2531,7 @@ TEST_F(OpTest, Shape) { TEST_F(OpTest, ShapeN) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(1, 5)(generator()); OpTestBuilder builder("ShapeN"); builder.Attr("T", type); @@ -2493,7 +2545,7 @@ TEST_F(OpTest, ShapeN) { TEST_F(OpTest, Sigmoid) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sigmoid").RandomInput(type).Attr("T", type)); }); @@ -2502,7 +2554,7 @@ TEST_F(OpTest, Sigmoid) { TEST_F(OpTest, SigmoidGrad) { Repeatedly([this]() { auto dims = RandomDims(); - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SigmoidGrad") .RandomInput(type, dims) .RandomInput(type, dims) @@ -2512,7 +2564,7 @@ TEST_F(OpTest, SigmoidGrad) { TEST_F(OpTest, Sign) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sign").RandomInput(type).Attr("T", type)); }); @@ -2520,7 +2572,7 @@ TEST_F(OpTest, Sign) { TEST_F(OpTest, Sin) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sin").RandomInput(type).Attr("T", type)); }); @@ -2528,7 +2580,7 @@ TEST_F(OpTest, Sin) { TEST_F(OpTest, Sinh) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sinh").RandomInput(type).Attr("T", type)); }); @@ -2536,7 +2588,7 @@ TEST_F(OpTest, Sinh) { TEST_F(OpTest, Size) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Size").RandomInput(type).Attr("T", type)); }); @@ -2544,7 +2596,7 @@ TEST_F(OpTest, Size) { TEST_F(OpTest, Slice) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); std::vector begin(data_dims.size()), size(data_dims.size()); @@ -2648,7 +2700,7 @@ TEST_F(OpTest, SpaceToBatch) { CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals), TensorShape({num_block_dims, 2}))); - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SpaceToBatch") .RandomInput(type, input_dims) .Input(paddings) @@ -2690,7 +2742,7 @@ TEST_F(OpTest, SpaceToBatchND) { CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals), TensorShape({num_block_dims, 2}))); - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("SpaceToBatchND") .RandomInput(type, input_dims) @@ -2767,7 +2819,7 @@ TEST_F(OpTest, SparseSoftmaxCrossEntropyWithLogits) { TEST_F(OpTest, Split) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(1); std::uniform_int_distribution ud; int32 dim = std::uniform_int_distribution( @@ -2787,7 +2839,7 @@ TEST_F(OpTest, Split) { TEST_F(OpTest, Sqrt) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sqrt").RandomInput(type).Attr("T", type)); }); @@ -2796,7 +2848,7 @@ TEST_F(OpTest, Sqrt) { TEST_F(OpTest, SqrtGrad) { Repeatedly([this]() { auto dims = RandomDims(); - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SqrtGrad") .RandomInput(type, dims) .RandomInput(type, dims) @@ -2816,7 +2868,7 @@ TEST_F(OpTest, SquaredDifference) { TEST_F(OpTest, Square) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Square").RandomInput(type).Attr("T", type)); }); @@ -2824,7 +2876,7 @@ TEST_F(OpTest, Square) { TEST_F(OpTest, Squeeze) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector t_dims = RandomDims(0, kDefaultMaxRank, 0, 5); std::bernoulli_distribution random_bool; std::vector squeeze_dims; @@ -2842,7 +2894,7 @@ TEST_F(OpTest, Squeeze) { TEST_F(OpTest, Sub) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Sub") .RandomInput(type, dims.first) @@ -2853,7 +2905,7 @@ TEST_F(OpTest, Sub) { TEST_F(OpTest, Sum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2867,7 +2919,7 @@ TEST_F(OpTest, Sum) { TEST_F(OpTest, StridedSlice) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); std::vector begin(data_dims.size()), end(data_dims.size()); std::vector strides(data_dims.size()); @@ -2912,7 +2964,7 @@ TEST_F(OpTest, StridedSlice) { TEST_F(OpTest, StridedSliceGrad) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); // Dimensions of the forward input. std::vector dims = RandomDims(); @@ -2965,7 +3017,7 @@ TEST_F(OpTest, StridedSliceGrad) { TEST_F(OpTest, Tan) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Tan").RandomInput(type).Attr("T", type)); }); @@ -2973,7 +3025,7 @@ TEST_F(OpTest, Tan) { TEST_F(OpTest, Tanh) { Repeatedly([this]() { - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Tanh").RandomInput(type).Attr("T", type)); }); @@ -2982,7 +3034,7 @@ TEST_F(OpTest, Tanh) { TEST_F(OpTest, TanhGrad) { Repeatedly([this]() { auto dims = RandomDims(); - DataType type = Choose({DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TanhGrad") .RandomInput(type, dims) .RandomInput(type, dims) @@ -2992,7 +3044,7 @@ TEST_F(OpTest, TanhGrad) { TEST_F(OpTest, Tile) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector t_dims = RandomDims(1); std::vector multiples(t_dims.size()); for (int i = 0; i < t_dims.size(); ++i) { @@ -3008,7 +3060,7 @@ TEST_F(OpTest, Tile) { TEST_F(OpTest, Transpose) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); std::vector perm(data_dims.size()); std::iota(perm.begin(), perm.end(), 0); @@ -3033,7 +3085,7 @@ TEST_F(OpTest, TruncateDiv) { TEST_F(OpTest, TruncateMod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TruncateMod") .RandomInput(type, dims.first) @@ -3044,7 +3096,7 @@ TEST_F(OpTest, TruncateMod) { TEST_F(OpTest, ZerosLike) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("ZerosLike").RandomInput(type).Attr("T", type)); }); diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc index b35f6fc2e0..a266e9013c 100644 --- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc @@ -45,7 +45,7 @@ XLAJIT_MAKE_UNARY(ComplexAbs, b->Abs(x)); XLAJIT_MAKE_UNARY(Angle, b->Atan2(b->Imag(x), b->Real(x))); -XLAJIT_MAKE_UNARY(Conj, b->Complex(b->Real(x), b->Neg(b->Imag(x)))); +XLAJIT_MAKE_UNARY(Conj, b->Conj(x)); // Return x if x>0, otherwise -x. XLAJIT_MAKE_UNARY(Abs, b->Abs(x)); -- GitLab From e9af1af4f2ca7dd0d767ff60cb08034f37de44ad Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Fri, 27 Oct 2017 13:36:53 -0700 Subject: [PATCH 475/573] Fixing the sources docs in master. --- tensorflow/docs_src/install/install_sources.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index b853d87816..bef8b7ad02 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -455,11 +455,11 @@ Stack Overflow and specify the `tensorflow` tag.
Example Data Set Demonstrates How To...
Example Demonstrates How To...
linear_regression.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use the @{tf.estimator.LinearRegressor} Estimator to train a regression model on numeric data.
linear_regression_categorical.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use the @{tf.estimator.LinearRegressor} Estimator to train a regression model on categorical data.
dnn_regression.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use the @{tf.estimator.DNNRegressor} Estimator to train a regression model on discrete data with a deep neural network.
custom_regression.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use @{tf.estimator.Estimator} to train a customized dnn regression model.
- - - - - + + + + +
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
ttensorflow-1.2.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
ttensorflow-1.1.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
ttensorflow_gpu-1.1.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
ttensorflow-1.0.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
ttensorflow_gpu-1.0.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
tensorflow-1.2.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.1.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
tensorflow_gpu-1.1.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
tensorflow-1.0.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
tensorflow_gpu-1.0.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
**Windows** -- GitLab From 0bc432a443e70bcd38326291f9746f965ffd1de2 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 27 Oct 2017 13:39:08 -0700 Subject: [PATCH 476/573] TFE: Add compatibility errors and doc strings to queues, input pipelines and Supervisor PiperOrigin-RevId: 173712330 --- tensorflow/python/training/input.py | 74 ++++++++++++++++--- .../python/training/queue_runner_impl.py | 12 +++ tensorflow/python/training/supervisor.py | 20 +++++ 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index e7adbf11b4..f645d8cf39 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -149,14 +149,15 @@ def input_producer(input_tensor, RuntimeError: If called with eager execution enabled. @compatibility(eager) - Queue-using input pipelines are not supported when eager execution is enabled. - Please use tf.data to ingest data into your model instead. + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. @end_compatibility """ if context.in_eager_mode(): raise RuntimeError( - "Queue-using input pipelines are not supported when eager execution is" - " enabled. Please use tf.data to ingest data into your model instead.") + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") with ops.name_scope(name, "input_producer", [input_tensor]): input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor") element_shape = input_tensor.shape[1:].merge_with(element_shape) @@ -222,6 +223,11 @@ def string_input_producer(string_tensor, Raises: ValueError: If the string_tensor is a null Python list. At runtime, will fail with an assertion if string_tensor becomes a null tensor. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ not_null_err = "string_input_producer requires a non-null input tensor" if not isinstance(string_tensor, ops.Tensor) and not string_tensor: @@ -271,6 +277,11 @@ def range_input_producer(limit, num_epochs=None, shuffle=True, seed=None, Returns: A Queue with the output integers. A `QueueRunner` for the Queue is added to the current `Graph`'s `QUEUE_RUNNER` collection. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ with ops.name_scope(name, "input_producer", [limit]) as name: range_tensor = math_ops.range(limit) @@ -308,6 +319,11 @@ def slice_input_producer(tensor_list, num_epochs=None, shuffle=True, seed=None, Raises: ValueError: if `slice_input_producer` produces nothing from `tensor_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ with ops.name_scope(name, "input_producer", tensor_list): tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensor_list) @@ -698,8 +714,9 @@ def _batch(tensors, batch_size, keep_input, num_threads=1, capacity=32, """Helper function for `batch` and `maybe_batch`.""" if context.in_eager_mode(): raise ValueError( - "Queue-using input pipelines are not supported when eager execution is" - " enabled. Please use tf.data to ingest data into your model instead.") + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list = _as_tensor_list(tensors) with ops.name_scope(name, "batch", list(tensor_list) + [keep_input]) as name: tensor_list = _validate(tensor_list) @@ -735,8 +752,9 @@ def _batch_join(tensors_list, batch_size, keep_input, capacity=32, """Helper function for `batch_join` and `maybe_batch_join`.""" if context.in_eager_mode(): raise ValueError( - "Queue-using input pipelines are not supported when eager execution is" - " enabled. Please use tf.data to ingest data into your model instead.") + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list_list = _as_tensor_list_list(tensors_list) with ops.name_scope(name, "batch_join", _flatten(tensor_list_list) + [keep_input]) as name: @@ -769,8 +787,9 @@ def _shuffle_batch(tensors, batch_size, capacity, min_after_dequeue, """Helper function for `shuffle_batch` and `maybe_shuffle_batch`.""" if context.in_eager_mode(): raise ValueError( - "Queue-using input pipelines are not supported when eager execution is" - " enabled. Please use tf.data to ingest data into your model instead.") + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list = _as_tensor_list(tensors) with ops.name_scope(name, "shuffle_batch", list(tensor_list) + [keep_input]) as name: @@ -813,8 +832,9 @@ def _shuffle_batch_join(tensors_list, batch_size, capacity, """Helper function for `shuffle_batch_join` and `maybe_shuffle_batch_join`.""" if context.in_eager_mode(): raise ValueError( - "Queue-using input pipelines are not supported when eager execution is" - " enabled. Please use tf.data to ingest data into your model instead.") + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list_list = _as_tensor_list_list(tensors_list) with ops.name_scope(name, "shuffle_batch_join", _flatten(tensor_list_list) + [keep_input]) as name: @@ -923,6 +943,11 @@ def batch(tensors, batch_size, num_threads=1, capacity=32, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _batch( tensors, @@ -1076,6 +1101,11 @@ def batch_join(tensors_list, batch_size, capacity=32, enqueue_many=False, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensor_list_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _batch_join( tensors_list, @@ -1220,6 +1250,11 @@ def shuffle_batch(tensors, batch_size, capacity, min_after_dequeue, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch( tensors, @@ -1274,6 +1309,11 @@ def maybe_shuffle_batch(tensors, batch_size, capacity, min_after_dequeue, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch( tensors, @@ -1363,6 +1403,11 @@ def shuffle_batch_join(tensors_list, batch_size, capacity, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch_join( tensors_list, @@ -1417,6 +1462,11 @@ def maybe_shuffle_batch_join(tensors_list, batch_size, capacity, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch_join( tensors_list, diff --git a/tensorflow/python/training/queue_runner_impl.py b/tensorflow/python/training/queue_runner_impl.py index 5abc6a2f58..d3b473ee46 100644 --- a/tensorflow/python/training/queue_runner_impl.py +++ b/tensorflow/python/training/queue_runner_impl.py @@ -23,6 +23,7 @@ import weakref from tensorflow.core.protobuf import queue_runner_pb2 from tensorflow.python.client import session +from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.platform import tf_logging as logging @@ -414,7 +415,18 @@ def start_queue_runners(sess=None, coord=None, daemon=True, start=True, Returns: A list of threads. + + Raises: + RuntimeError: If called with eager execution enabled. + ValueError: If called without a default `tf.Session` registered. + + @compatibility(eager) + Not compatible with eager execution. To ingest data under eager execution, + use the `tf.data` API instead. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError("Queues are not compatible with eager execution.") if sess is None: sess = ops.get_default_session() if not sess: diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index 41dbf6b497..a634a842b6 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -23,6 +23,7 @@ import time from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.core.util.event_pb2 import SessionLog +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops @@ -288,7 +289,16 @@ class Supervisor(object): Returns: A `Supervisor`. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + `Supervisor`s are not supported when eager execution is enabled. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError("Supervisors are compatible with eager execution.") # Set default values of arguments. if graph is None: graph = ops.get_default_graph() @@ -735,7 +745,17 @@ class Supervisor(object): Returns: The list of threads started for the `QueueRunners`. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Queues are not compatible with eager execution. To ingest data when eager + execution is enabled, use the `tf.data` API. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError("Queues are not compatible with eager execution.") if queue_runners is None: queue_runners = self._graph.get_collection(ops.GraphKeys.QUEUE_RUNNERS) threads = [] -- GitLab From 9bf00c3717d24fbc6bfad2a99c7c08f39f534aa3 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 27 Oct 2017 14:10:36 -0700 Subject: [PATCH 477/573] Shorter import for tfe. PiperOrigin-RevId: 173716375 --- tensorflow/contrib/__init__.py | 1 + tensorflow/contrib/cmake/tf_python.cmake | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index 76a629663d..a26fdb982c 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -77,6 +77,7 @@ from tensorflow.contrib import timeseries from tensorflow.contrib import tpu from tensorflow.contrib import training from tensorflow.contrib import util +from tensorflow.contrib.eager.python import tfe as eager from tensorflow.contrib.ndlstm import python as ndlstm from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph from tensorflow.contrib.specs import python as specs diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index a3ed19977f..1b9fd514fd 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -347,6 +347,8 @@ add_python_module("tensorflow/contrib/distributions/python") add_python_module("tensorflow/contrib/distributions/python/kernel_tests") add_python_module("tensorflow/contrib/distributions/python/ops") add_python_module("tensorflow/contrib/distributions/python/ops/bijectors") +add_python_module("tensorflow/contrib/eager") +add_python_module("tensorflow/contrib/eager/python") add_python_module("tensorflow/contrib/estimator") add_python_module("tensorflow/contrib/estimator/python") add_python_module("tensorflow/contrib/estimator/python/estimator") -- GitLab From 78bac7290c4c49c27ca61aa891ae564c54e2ddfc Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 27 Oct 2017 14:15:12 -0700 Subject: [PATCH 478/573] TFE: Add compatbility doc string to add_to_collection() and friends PiperOrigin-RevId: 173716912 --- tensorflow/python/framework/ops.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index c8ee9243d7..63f70a1a9d 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -4975,6 +4975,10 @@ def add_to_collection(name, value): name: The key for the collection. For example, the `GraphKeys` class contains many standard names for collections. value: The value to add to the collection. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ get_default_graph().add_to_collection(name, value) @@ -4989,6 +4993,10 @@ def add_to_collections(names, value): names: The key for the collections. The `GraphKeys` class contains many standard names for collections. value: The value to add to the collections. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ get_default_graph().add_to_collections(names, value) @@ -5008,6 +5016,10 @@ def get_collection_ref(key): list if no value has been added to that collection. Note that this returns the collection list itself, which can be modified in place to change the collection. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ return get_default_graph().get_collection_ref(key) @@ -5032,6 +5044,10 @@ def get_collection(key, scope=None): an empty list if no value has been added to that collection. The list contains the values in the order under which they were collected. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ return get_default_graph().get_collection(key, scope) -- GitLab From 02f55400f87b22f7ea0849c39022792d1e381afb Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 27 Oct 2017 15:08:01 -0700 Subject: [PATCH 479/573] custom_gradient functions should be able to return their inputs PiperOrigin-RevId: 173723462 --- tensorflow/python/eager/backprop_test.py | 12 ++++++++++++ tensorflow/python/eager/custom_gradient.py | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index d18df4dffb..20532c8ee8 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -569,5 +569,17 @@ class BackpropTest(test.TestCase): var.assign_sub(lr*grad) self.assertAllEqual(losses, [4.0, 3., 2., 1., 0.]) + def testCustomGradientIdentity(self): + + @custom_gradient.custom_gradient + def my_identity(x): + + def grad(dresult): + return [2 * dresult] + + return x, grad + + self.assertAllEqual(backprop.gradients_function(my_identity)(1.0)[0], 2.0) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py index 4ac30075b2..05460ff996 100644 --- a/tensorflow/python/eager/custom_gradient.py +++ b/tensorflow/python/eager/custom_gradient.py @@ -22,6 +22,7 @@ from tensorflow.python.eager import context from tensorflow.python.eager import tape from tensorflow.python.framework import ops as tf_ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -72,17 +73,19 @@ def custom_gradient(f): with tape.stop_recording(): result, grad_fn = f(*args, **kwargs) + flat_result = nest.flatten(result) + # TODO(apassos) consider removing the identity below. + flat_result = [gen_array_ops.identity(x) for x in flat_result] def actual_grad_fn(*outputs): return nest.flatten(grad_fn(*outputs)) - flat_result = nest.flatten(result) tape.record_operation( f.__name__, flat_result, input_tensors, actual_grad_fn) flat_result = list(flat_result) - return result + return nest.pack_sequence_as(result, flat_result) return tf_decorator.make_decorator(f, decorated) -- GitLab From 5426a3c93d8a180b7009ba87af12c61dc1a6278d Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Fri, 27 Oct 2017 15:35:06 -0700 Subject: [PATCH 480/573] Add tfe.get_optimizer_variables for fetching a list of variables which an optimizer has created. Useful for saving them if executing eagerly. PiperOrigin-RevId: 173726859 --- tensorflow/contrib/eager/python/saver.py | 23 +++++++++++ tensorflow/contrib/eager/python/saver_test.py | 41 +++++++++++++++++++ tensorflow/contrib/eager/python/tfe.py | 2 + 3 files changed, 66 insertions(+) diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py index d74e0fef3e..e0a20d2485 100644 --- a/tensorflow/contrib/eager/python/saver.py +++ b/tensorflow/contrib/eager/python/saver.py @@ -23,6 +23,7 @@ from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.training import adam as _adam from tensorflow.python.training import checkpoint_utils from tensorflow.python.training import saver as _saver @@ -165,3 +166,25 @@ class Saver(object): """ with ops.device("/device:CPU:0"): self._saver.restore(None, file_prefix) + + +def get_optimizer_variables(optimizer): + """Returns a list of variables for the given `tf.train.Optimizer`. + + Args: + optimizer: An instance of `tf.train.Optimizer` which has created variables + (typically after a call to `Optimizer.minimize`). + Returns: + A list of variables which have been created by the `Optimizer`. Currently + returns all variables even if they were not created in the default graph, + but this behavior may change. + """ + variables = [] + # pylint: disable=protected-access + for _, variable_dict in optimizer._slots.items(): + for _, slot_for_variable in variable_dict.items(): + variables.append(slot_for_variable) + if isinstance(optimizer, _adam.AdamOptimizer): + variables.append(optimizer._beta1_power) + variables.append(optimizer._beta2_power) + return variables diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py index 3c69b90242..abc7e3690c 100644 --- a/tensorflow/contrib/eager/python/saver_test.py +++ b/tensorflow/contrib/eager/python/saver_test.py @@ -30,6 +30,10 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope +from tensorflow.python.training import adam +from tensorflow.python.training import gradient_descent +from tensorflow.python.training import momentum +from tensorflow.python.training import rmsprop class SaverTest(test.TestCase): @@ -204,5 +208,42 @@ class SaverTest(test.TestCase): 3, model2(array_ops.constant(2, dtype=dtypes.float32)).numpy()) +class GetOptimizerTests(test.TestCase): + + def _optimizer_test_template(self, optimizer): + """Checks save and restore. Returns the optimizer variables.""" + v = resource_variable_ops.ResourceVariable([[2., 3.]], name='v') + loss_fn = lambda: v[0, 0] ** 2 + v[0, 1] ** 2 + optimizer.minimize(loss_fn) + optimizer_variables = _saver.get_optimizer_variables(optimizer) + saver = _saver.Saver(optimizer_variables + [v]) + checkpoint_path = saver.save(self.get_temp_dir()) + optimizer.minimize(loss_fn) + after_first_minimize = v.numpy() + # After we restore, the next step should be exactly the same as the one we + # just did. + saver.restore(checkpoint_path) + optimizer.minimize(loss_fn) + self.assertAllEqual(after_first_minimize, v.numpy()) + return optimizer_variables + + def testAdam(self): + optimizer = adam.AdamOptimizer(0.1) + self._optimizer_test_template(optimizer) + + def testGradientDescent(self): + optimizer = gradient_descent.GradientDescentOptimizer(0.02) + self.assertEqual(0, len(self._optimizer_test_template(optimizer))) + + def testMomentum(self): + optimizer = momentum.MomentumOptimizer( + learning_rate=0.03, + momentum=0.5) + self._optimizer_test_template(optimizer) + + def testRMSProp(self): + optimizer = rmsprop.RMSPropOptimizer(0.01) + self._optimizer_test_template(optimizer) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index a769140713..ab31893cd3 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -51,6 +51,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@SummaryWriter @@restore_variables_on_create @@Variable +@@get_optimizer_variables @@in_eager_mode @@in_graph_mode @@ -73,6 +74,7 @@ from __future__ import print_function from tensorflow.contrib.eager.python import metrics from tensorflow.contrib.eager.python.datasets import Iterator from tensorflow.contrib.eager.python.network import Network +from tensorflow.contrib.eager.python.saver import get_optimizer_variables from tensorflow.contrib.eager.python.saver import restore_variables_on_create from tensorflow.contrib.eager.python.saver import Saver from tensorflow.contrib.eager.python.summary_writer import SummaryWriter -- GitLab From 80374a7b47dddb591f711b6240ea0896fbe90d29 Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Fri, 27 Oct 2017 15:59:46 -0700 Subject: [PATCH 481/573] Breaking change: Rename `tf.contrib.distributions.Independent` parameter from `reduce_batch_ndims` to `reinterpreted_batch_ndims`. Also change default; `reinterpreted_batch_ndims` default has semantics of `tf.layers.flatten`, i.e., all batch dimensions except the first (batch axis 0) are interpretted as being part of the event. PiperOrigin-RevId: 173729585 --- .../python/kernel_tests/independent_test.py | 65 +++++++++- .../distributions/python/ops/independent.py | 113 +++++++++++------- 2 files changed, 130 insertions(+), 48 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py index dcc66e8972..8e23a3ab8f 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py @@ -23,7 +23,10 @@ import numpy as np from tensorflow.contrib.distributions.python.ops import independent as independent_lib from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bernoulli as bernoulli_lib from tensorflow.python.ops.distributions import normal as normal_lib from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging @@ -42,13 +45,16 @@ stats = try_import("scipy.stats") class ProductDistributionTest(test.TestCase): + def setUp(self): + self._rng = np.random.RandomState(42) + def testSampleAndLogProbUnivariate(self): loc = np.float32([-1., 1]) scale = np.float32([0.1, 0.5]) with self.test_session() as sess: ind = independent_lib.Independent( distribution=normal_lib.Normal(loc=loc, scale=scale), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) x = ind.sample([4, 5]) log_prob_x = ind.log_prob(x) @@ -71,7 +77,7 @@ class ProductDistributionTest(test.TestCase): distribution=mvn_diag_lib.MultivariateNormalDiag( loc=loc, scale_identity_multiplier=scale), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) x = ind.sample([4, 5]) log_prob_x = ind.log_prob(x) @@ -96,7 +102,7 @@ class ProductDistributionTest(test.TestCase): distribution=mvn_diag_lib.MultivariateNormalDiag( loc=loc, scale_identity_multiplier=scale), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) x = ind.sample(int(n_samp), seed=42) sample_mean = math_ops.reduce_mean(x, axis=0) @@ -120,6 +126,59 @@ class ProductDistributionTest(test.TestCase): self.assertAllClose(sample_entropy_, actual_entropy_, rtol=0.01, atol=0.) self.assertAllClose(loc, actual_mode_, rtol=1e-6, atol=0.) + def _testMnistLike(self, static_shape): + sample_shape = [4, 5] + batch_shape = [10] + image_shape = [28, 28, 1] + logits = 3 * self._rng.random_sample( + batch_shape + image_shape).astype(np.float32) - 1 + + def expected_log_prob(x, logits): + return (x * logits - np.log1p(np.exp(logits))).sum(-1).sum(-1).sum(-1) + + with self.test_session() as sess: + logits_ph = array_ops.placeholder( + dtypes.float32, shape=logits.shape if static_shape else None) + ind = independent_lib.Independent( + distribution=bernoulli_lib.Bernoulli(logits=logits_ph)) + x = ind.sample(sample_shape) + log_prob_x = ind.log_prob(x) + [ + x_, + actual_log_prob_x, + ind_batch_shape, + ind_event_shape, + x_shape, + log_prob_x_shape, + ] = sess.run([ + x, + log_prob_x, + ind.batch_shape_tensor(), + ind.event_shape_tensor(), + array_ops.shape(x), + array_ops.shape(log_prob_x), + ], feed_dict={logits_ph: logits}) + + if static_shape: + ind_batch_shape = ind.batch_shape + ind_event_shape = ind.event_shape + x_shape = x.shape + log_prob_x_shape = log_prob_x.shape + + self.assertAllEqual(batch_shape, ind_batch_shape) + self.assertAllEqual(image_shape, ind_event_shape) + self.assertAllEqual(sample_shape + batch_shape + image_shape, x_shape) + self.assertAllEqual(sample_shape + batch_shape, log_prob_x_shape) + self.assertAllClose(expected_log_prob(x_, logits), + actual_log_prob_x, + rtol=1e-6, atol=0.) + + def testMnistLikeStaticShape(self): + self._testMnistLike(static_shape=True) + + def testMnistLikeDynamicShape(self): + self._testMnistLike(static_shape=False) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py index 393c008242..6a74ca9a0a 100644 --- a/tensorflow/contrib/distributions/python/ops/independent.py +++ b/tensorflow/contrib/distributions/python/ops/independent.py @@ -45,24 +45,24 @@ class Independent(distribution_lib.Distribution): `p(x_1, ..., x_B) = p_1(x_1) * ... * p_B(x_B)` where `p_b(X_b)` is the probability of the `b`-th rv. More generally `B, E` can be arbitrary shapes. - Similarly, the `Independent` distribution specifies a distribution over - `[B, E]`-shaped events. It operates by reinterpreting the rightmost batch dims - as part of the event dimensions. The `reduce_batch_ndims` parameter controls - the number of batch dims which are absorbed as event dims; - `reduce_batch_ndims < len(batch_shape)`. For example, the `log_prob` function - entails a `reduce_sum` over the rightmost `reduce_batch_ndims` after calling - the base distribution's `log_prob`. In other words, since the batch - dimension(s) index independent distributions, the resultant multivariate will - have independent components. + Similarly, the `Independent` distribution specifies a distribution over `[B, + E]`-shaped events. It operates by reinterpreting the rightmost batch dims as + part of the event dimensions. The `reinterpreted_batch_ndims` parameter + controls the number of batch dims which are absorbed as event dims; + `reinterpreted_batch_ndims < len(batch_shape)`. For example, the `log_prob` + function entails a `reduce_sum` over the rightmost `reinterpreted_batch_ndims` + after calling the base distribution's `log_prob`. In other words, since the + batch dimension(s) index independent distributions, the resultant multivariate + will have independent components. #### Mathematical Details The probability function is, ```none - prob(x; reduce_batch_ndims) = tf.reduce_prod( + prob(x; reinterpreted_batch_ndims) = tf.reduce_prod( dist.prob(x), - axis=-1-range(reduce_batch_ndims)) + axis=-1-range(reinterpreted_batch_ndims)) ``` #### Examples @@ -73,7 +73,7 @@ class Independent(distribution_lib.Distribution): # Make independent distribution from a 2-batch Normal. ind = ds.Independent( distribution=ds.Normal(loc=[-1., 1], scale=[0.1, 0.5]), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) # All batch dims have been "absorbed" into event dims. ind.batch_shape # ==> [] @@ -84,7 +84,7 @@ class Independent(distribution_lib.Distribution): distribution=ds.MultivariateNormalDiag( loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) # All batch dims have been "absorbed" into event dims. ind.batch_shape # ==> [] @@ -94,14 +94,17 @@ class Independent(distribution_lib.Distribution): """ def __init__( - self, distribution, reduce_batch_ndims=1, validate_args=False, name=None): + self, distribution, reinterpreted_batch_ndims=None, + validate_args=False, name=None): """Construct a `Independent` distribution. Args: distribution: The base distribution instance to transform. Typically an instance of `Distribution`. - reduce_batch_ndims: Scalar, integer number of rightmost batch dims which - will be regard as event dims. + reinterpreted_batch_ndims: Scalar, integer number of rightmost batch dims + which will be regarded as event dims. When `None` all but the first + batch axis (batch axis 0) will be transferred to event dimensions + (analogous to `tf.layers.flatten`). validate_args: Python `bool`. Whether to validate input with asserts. If `validate_args` is `False`, and the inputs are invalid, correct behavior is not guaranteed. @@ -109,19 +112,25 @@ class Independent(distribution_lib.Distribution): Default value: `Independent + distribution.name`. Raises: - ValueError: if `reduce_batch_ndims` exceeds `distribution.batch_ndims` + ValueError: if `reinterpreted_batch_ndims` exceeds + `distribution.batch_ndims` """ parameters = locals() name = name or "Independent" + distribution.name self._distribution = distribution with ops.name_scope(name): - reduce_batch_ndims = ops.convert_to_tensor( - reduce_batch_ndims, dtype=dtypes.int32, name="reduce_batch_ndims") - self._reduce_batch_ndims = reduce_batch_ndims - self._static_reduce_batch_ndims = tensor_util.constant_value( - reduce_batch_ndims) - if self._static_reduce_batch_ndims is not None: - self._reduce_batch_ndims = self._static_reduce_batch_ndims + if reinterpreted_batch_ndims is None: + reinterpreted_batch_ndims = self._get_default_reinterpreted_batch_ndims( + distribution) + reinterpreted_batch_ndims = ops.convert_to_tensor( + reinterpreted_batch_ndims, + dtype=dtypes.int32, + name="reinterpreted_batch_ndims") + self._reinterpreted_batch_ndims = reinterpreted_batch_ndims + self._static_reinterpreted_batch_ndims = tensor_util.constant_value( + reinterpreted_batch_ndims) + if self._static_reinterpreted_batch_ndims is not None: + self._reinterpreted_batch_ndims = self._static_reinterpreted_batch_ndims super(Independent, self).__init__( dtype=self._distribution.dtype, reparameterization_type=self._distribution.reparameterization_type, @@ -129,19 +138,19 @@ class Independent(distribution_lib.Distribution): allow_nan_stats=self._distribution.allow_nan_stats, parameters=parameters, graph_parents=( - [reduce_batch_ndims] + + [reinterpreted_batch_ndims] + distribution._graph_parents), # pylint: disable=protected-access name=name) self._runtime_assertions = self._make_runtime_assertions( - distribution, reduce_batch_ndims, validate_args) + distribution, reinterpreted_batch_ndims, validate_args) @property def distribution(self): return self._distribution @property - def reduce_batch_ndims(self): - return self._reduce_batch_ndims + def reinterpreted_batch_ndims(self): + return self._reinterpreted_batch_ndims def _batch_shape_tensor(self): with ops.control_dependencies(self._runtime_assertions): @@ -149,13 +158,14 @@ class Independent(distribution_lib.Distribution): batch_ndims = (batch_shape.shape[0].value if batch_shape.shape.with_rank_at_least(1)[0].value else array_ops.shape(batch_shape)[0]) - return batch_shape[:batch_ndims - self.reduce_batch_ndims] + return batch_shape[:batch_ndims - self.reinterpreted_batch_ndims] def _batch_shape(self): batch_shape = self.distribution.batch_shape - if self._static_reduce_batch_ndims is None or batch_shape.ndims is None: + if (self._static_reinterpreted_batch_ndims is None + or batch_shape.ndims is None): return tensor_shape.TensorShape(None) - d = batch_shape.ndims - self._static_reduce_batch_ndims + d = batch_shape.ndims - self._static_reinterpreted_batch_ndims return batch_shape[:d] def _event_shape_tensor(self): @@ -165,15 +175,16 @@ class Independent(distribution_lib.Distribution): if batch_shape.shape.with_rank_at_least(1)[0].value else array_ops.shape(batch_shape)[0]) return array_ops.concat([ - batch_shape[batch_ndims - self.reduce_batch_ndims:], + batch_shape[batch_ndims - self.reinterpreted_batch_ndims:], self.distribution.event_shape_tensor(), ], axis=0) def _event_shape(self): batch_shape = self.distribution.batch_shape - if self._static_reduce_batch_ndims is None or batch_shape.ndims is None: + if (self._static_reinterpreted_batch_ndims is None + or batch_shape.ndims is None): return tensor_shape.TensorShape(None) - d = batch_shape.ndims - self._static_reduce_batch_ndims + d = batch_shape.ndims - self._static_reinterpreted_batch_ndims return batch_shape[d:].concatenate(self.distribution.event_shape) def _sample_n(self, n, seed): @@ -205,15 +216,16 @@ class Independent(distribution_lib.Distribution): return self.distribution.mode() def _make_runtime_assertions( - self, distribution, reduce_batch_ndims, validate_args): + self, distribution, reinterpreted_batch_ndims, validate_args): assertions = [] - static_reduce_batch_ndims = tensor_util.constant_value(reduce_batch_ndims) + static_reinterpreted_batch_ndims = tensor_util.constant_value( + reinterpreted_batch_ndims) batch_ndims = distribution.batch_shape.ndims - if batch_ndims is not None and static_reduce_batch_ndims is not None: - if static_reduce_batch_ndims > batch_ndims: - raise ValueError("reduce_batch_ndims({}) cannot exceed " + if batch_ndims is not None and static_reinterpreted_batch_ndims is not None: + if static_reinterpreted_batch_ndims > batch_ndims: + raise ValueError("reinterpreted_batch_ndims({}) cannot exceed " "distribution.batch_ndims({})".format( - static_reduce_batch_ndims, batch_ndims)) + static_reinterpreted_batch_ndims, batch_ndims)) elif validate_args: batch_shape = distribution.batch_shape_tensor() batch_ndims = ( @@ -221,13 +233,24 @@ class Independent(distribution_lib.Distribution): if batch_shape.shape.with_rank_at_least(1)[0].value is not None else array_ops.shape(batch_shape)[0]) assertions.append(check_ops.assert_less_equal( - reduce_batch_ndims, batch_ndims, - message="reduce_batch_ndims cannot exceed distribution.batch_ndims")) + reinterpreted_batch_ndims, batch_ndims, + message=("reinterpreted_batch_ndims cannot exceed " + "distribution.batch_ndims"))) return assertions def _reduce_sum(self, stat): - if self._static_reduce_batch_ndims is None: - range_ = array_ops.range(self._reduce_batch_ndims) + if self._static_reinterpreted_batch_ndims is None: + range_ = math_ops.range(self._reinterpreted_batch_ndims) else: - range_ = np.arange(self._static_reduce_batch_ndims) + range_ = np.arange(self._static_reinterpreted_batch_ndims) return math_ops.reduce_sum(stat, axis=-1-range_) + + def _get_default_reinterpreted_batch_ndims(self, distribution): + """Computes the default value for reinterpreted_batch_ndim __init__ arg.""" + ndims = distribution.batch_shape.ndims + if ndims is None: + which_maximum = math_ops.maximum + ndims = array_ops.shape(distribution.batch_shape_tensor())[0] + else: + which_maximum = np.maximum + return which_maximum(0, ndims - 1) -- GitLab From e1d7615ebcad6f45b41f7849bc77a8aae17b8690 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 27 Oct 2017 16:10:54 -0700 Subject: [PATCH 482/573] Fix issue with gradients of functions which return multiple values. PiperOrigin-RevId: 173730922 --- tensorflow/python/eager/backprop.py | 34 +++++++++++++++++++----- tensorflow/python/eager/backprop_test.py | 7 +++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 6ede02dbcd..be733405a3 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -35,6 +35,7 @@ from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.util import nest @@ -322,7 +323,10 @@ def implicit_val_and_grad(f): ``` Args: - f: The function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. Returns: A function which, when called, returns a tuple pair. @@ -384,7 +388,10 @@ def implicit_grad(f): ``` Args: - f: The function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. Returns: A function which, when called, returns a list of (gradient, variable) pairs. @@ -467,7 +474,12 @@ def gradients_function(f, params=None): ``` Args: - f: function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. If desired, the tensors can be elementwise multiplied by the + tensors passed as the `dy` keyword argument to the returned gradient + function. params: list of parameter names of f or list of integers indexing the parameters with respect to which we'll differentiate. Passing None differentiates with respect to all parameters. @@ -559,7 +571,12 @@ def val_and_grad_function(f, params=None): ``` Args: - f: function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. If desired, the tensors can be elementwise multiplied by the + tensors passed as the `dy` keyword argument to the returned gradient + function. params: list of parameter names of f or list of integers indexing the parameters with respect to which we'll differentiate. Passing `None` differentiates with respect to all parameters. @@ -632,12 +649,17 @@ def make_vjp(f, params=None): sources.append(args[i]) tape.watch(args[i]) result = f(*args) + flat_result = nest.flatten(result) + flat_result = [gen_array_ops.identity(x) for x in flat_result] + result = nest.pack_sequence_as(result, flat_result) finally: t = tape.pop_tape() def vjp(dy=None): + if dy is not None: + dy = [ops.convert_to_tensor(x) for x in nest.flatten(dy)] return imperative_grad.imperative_grad( _default_vspace, t, nest.flatten(result), sources, - output_gradients=nest.flatten(dy) if dy is not None else None) + output_gradients=dy) return result, vjp return decorated @@ -697,7 +719,7 @@ _default_vspace = imperative_grad.VSpace( aggregate_fn=_aggregate_grads, tensor_id=ops.tensor_id, zeros=array_ops.zeros, - ones_like=array_ops.ones_like) + ones_like=lambda x: ops.convert_to_tensor(array_ops.ones_like(x))) class GradientTape(object): diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 20532c8ee8..cf736fcb13 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -401,6 +401,13 @@ class BackpropTest(test.TestCase): backprop.gradients_function(part)(constant_op.constant(1.0))[0], 2.0) + def testReturnSameThing(self): + + def f(x): + return x, 2 * x + + self.assertAllEqual(backprop.gradients_function(f)(1.0)[0], 3.0) + def testExceptionSafety(self): def f(unused_x): -- GitLab From 7cb7f88c5ff12a5ce52e82d9f07e1b489df1e0ff Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 16:11:56 -0700 Subject: [PATCH 483/573] Add count metric, a helper function that computes the total number or total weight of examples. PiperOrigin-RevId: 173731046 --- tensorflow/contrib/metrics/__init__.py | 2 + .../contrib/metrics/python/ops/metric_ops.py | 80 ++++++++- .../metrics/python/ops/metric_ops_test.py | 158 ++++++++++++++++++ 3 files changed, 231 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py index 2c48882d0e..bb566f6902 100644 --- a/tensorflow/contrib/metrics/__init__.py +++ b/tensorflow/contrib/metrics/__init__.py @@ -65,6 +65,7 @@ See the @{$python/contrib.metrics} guide. @@set_intersection @@set_size @@set_union +@@count """ from __future__ import absolute_import @@ -78,6 +79,7 @@ from tensorflow.contrib.metrics.python.ops.confusion_matrix_ops import confusion from tensorflow.contrib.metrics.python.ops.histogram_ops import auc_using_histogram from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metric_map from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metrics +from tensorflow.contrib.metrics.python.ops.metric_ops import count from tensorflow.contrib.metrics.python.ops.metric_ops import sparse_recall_at_top_k from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_accuracy from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 50b9c4afde..177c4c53f7 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -120,7 +120,7 @@ def _count_condition(values, or tuple. """ check_ops.assert_type(values, dtypes.bool) - count = _create_local('count', shape=[]) + count_ = _create_local('count', shape=[]) values = math_ops.to_float(values) if weights is not None: @@ -128,8 +128,8 @@ def _count_condition(values, with ops.control_dependencies((_assert_weights_rank(weights, values),)): values = math_ops.multiply(values, weights) - value_tensor = array_ops.identity(count) - update_op = state_ops.assign_add(count, math_ops.reduce_sum(values)) + value_tensor = array_ops.identity(count_) + update_op = state_ops.assign_add(count_, math_ops.reduce_sum(values)) if metrics_collections: ops.add_to_collections(metrics_collections, value_tensor) @@ -2601,7 +2601,7 @@ def streaming_covariance(predictions, predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - count = _create_local('count', []) + count_ = _create_local('count', []) mean_prediction = _create_local('mean_prediction', []) mean_label = _create_local('mean_label', []) comoment = _create_local('comoment', []) # C_A in update equation @@ -2616,7 +2616,7 @@ def streaming_covariance(predictions, weighted_predictions = math_ops.multiply(predictions, weights) weighted_labels = math_ops.multiply(labels, weights) - update_count = state_ops.assign_add(count, batch_count) # n_AB in eqn + update_count = state_ops.assign_add(count_, batch_count) # n_AB in eqn prev_count = update_count - batch_count # n_A in update equation # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount) @@ -2660,15 +2660,15 @@ def streaming_covariance(predictions, update_comoment = state_ops.assign_add(comoment, delta_comoment) covariance = array_ops.where( - math_ops.less_equal(count, 1.), + math_ops.less_equal(count_, 1.), float('nan'), - math_ops.truediv(comoment, count - 1), + math_ops.truediv(comoment, count_ - 1), name='covariance') with ops.control_dependencies([update_comoment]): update_op = array_ops.where( - math_ops.less_equal(count, 1.), + math_ops.less_equal(count_, 1.), float('nan'), - math_ops.truediv(comoment, count - 1), + math_ops.truediv(comoment, count_ - 1), name='update_op') if metrics_collections: @@ -3124,9 +3124,71 @@ def aggregate_metric_map(names_to_tuples): return dict(zip(metric_names, value_ops)), dict(zip(metric_names, update_ops)) +def count(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): + """Computes the number of examples, or sum of `weights`. + + When evaluating some metric (e.g. mean) on one or more subsets of the data, + this auxiliary metric is useful for keeping track of how many examples there + are in each subset. + + If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. + + Args: + values: A `Tensor` of arbitrary dimensions. Only it's shape is used. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions + must be either `1`, or the same as the corresponding `labels` + dimension). + metrics_collections: An optional list of collections that the metric + value variable should be added to. + updates_collections: An optional list of collections that the metric update + ops should be added to. + name: An optional variable_scope name. + + Returns: + count: A `Tensor` representing the current value of the metric. + update_op: An operation that accumulates the metric from a batch of data. + + Raises: + ValueError: If `weights` is not `None` and its shape doesn't match `values`, + or if either `metrics_collections` or `updates_collections` are not a list + or tuple. + """ + + with variable_scope.variable_scope(name, 'count', (values, weights)): + count_ = _create_local('count', shape=[]) + + if weights is None: + num_values = math_ops.to_float(array_ops.size(values)) + else: + _, _, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access + predictions=values, + labels=None, + weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.to_float(weights), values) + num_values = math_ops.reduce_sum(weights) + + with ops.control_dependencies([values]): + update_op = state_ops.assign_add(count_, num_values) + + if metrics_collections: + ops.add_to_collections(metrics_collections, count_) + + if updates_collections: + ops.add_to_collections(updates_collections, update_op) + + return count_, update_op + + __all__ = [ 'aggregate_metric_map', 'aggregate_metrics', + 'count', 'sparse_recall_at_top_k', 'streaming_accuracy', 'streaming_auc', diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py index 24d82a7eee..6a8284786f 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py @@ -6170,5 +6170,163 @@ class AggregateMetricMapTest(test.TestCase): self.assertEqual(4, names_to_values['m2'].eval()) +class CountTest(test.TestCase): + + def setUp(self): + ops.reset_default_graph() + + def testVars(self): + metrics.count(array_ops.ones([4, 3])) + _assert_local_variables(self, ['count/count:0']) + + def testMetricsCollection(self): + my_collection_name = '__metrics__' + mean, _ = metrics.count( + array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) + self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + + def testUpdatesCollection(self): + my_collection_name = '__updates__' + _, update_op = metrics.count( + array_ops.ones([4, 3]), updates_collections=[my_collection_name]) + self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + + def testBasic(self): + with self.test_session() as sess: + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + result, update_op = metrics.count(values) + + sess.run(variables.local_variables_initializer()) + for _ in range(4): + sess.run(update_op) + self.assertAlmostEqual(8.0, sess.run(result), 5) + + def testUpdateOpsReturnsCurrentValue(self): + with self.test_session() as sess: + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + result, update_op = metrics.count(values) + + sess.run(variables.local_variables_initializer()) + + self.assertAlmostEqual(2.0, sess.run(update_op), 5) + self.assertAlmostEqual(4.0, sess.run(update_op), 5) + self.assertAlmostEqual(6.0, sess.run(update_op), 5) + self.assertAlmostEqual(8.0, sess.run(update_op), 5) + + self.assertAlmostEqual(8.0, sess.run(result), 5) + + def test1dWeightedValues(self): + with self.test_session() as sess: + # Create the queue that populates the values. + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) + _enqueue_vector(sess, weights_queue, [0.5]) + _enqueue_vector(sess, weights_queue, [0]) + _enqueue_vector(sess, weights_queue, [0]) + _enqueue_vector(sess, weights_queue, [1.2]) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for _ in range(4): + update_op.eval() + self.assertAlmostEqual(3.4, result.eval(), 5) + + def test1dWeightedValues_placeholders(self): + with self.test_session() as sess: + # Create the queue that populates the values. + feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) + values = array_ops.placeholder(dtype=dtypes_lib.float32) + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1,)) + _enqueue_vector(sess, weights_queue, 0.5, shape=(1,)) + _enqueue_vector(sess, weights_queue, 0, shape=(1,)) + _enqueue_vector(sess, weights_queue, 0, shape=(1,)) + _enqueue_vector(sess, weights_queue, 1.2, shape=(1,)) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for i in range(4): + update_op.eval(feed_dict={values: feed_values[i]}) + self.assertAlmostEqual(3.4, result.eval(), 5) + + def test2dWeightedValues(self): + with self.test_session() as sess: + # Create the queue that populates the values. + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, weights_queue, [1.1, 1]) + _enqueue_vector(sess, weights_queue, [1, 0]) + _enqueue_vector(sess, weights_queue, [0, 1]) + _enqueue_vector(sess, weights_queue, [0, 0]) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for _ in range(4): + update_op.eval() + self.assertAlmostEqual(4.1, result.eval(), 5) + + def test2dWeightedValues_placeholders(self): + with self.test_session() as sess: + # Create the queue that populates the values. + feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) + values = array_ops.placeholder(dtype=dtypes_lib.float32) + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(2,)) + _enqueue_vector(sess, weights_queue, [1.1, 1], shape=(2,)) + _enqueue_vector(sess, weights_queue, [1, 0], shape=(2,)) + _enqueue_vector(sess, weights_queue, [0, 1], shape=(2,)) + _enqueue_vector(sess, weights_queue, [0, 0], shape=(2,)) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for i in range(4): + update_op.eval(feed_dict={values: feed_values[i]}) + self.assertAlmostEqual(4.1, result.eval(), 5) + + if __name__ == '__main__': test.main() -- GitLab From 46a577febccf874b4b8bb8d42be0a3fb069e380d Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Fri, 27 Oct 2017 16:25:49 -0700 Subject: [PATCH 484/573] [CMake] Generate audio_ops wrappers in the CMake build. Fixes #14004. PiperOrigin-RevId: 173732397 --- tensorflow/contrib/cmake/tf_core_ops.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake index 97bec81e66..15e9a4c461 100644 --- a/tensorflow/contrib/cmake/tf_core_ops.cmake +++ b/tensorflow/contrib/cmake/tf_core_ops.cmake @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== set(tf_op_lib_names + "audio_ops" "array_ops" "bitwise_ops" "candidate_sampling_ops" -- GitLab From abbab2430cc5e3ef8eab224957c5cbfc8bd0056a Mon Sep 17 00:00:00 2001 From: Michael Case Date: Fri, 27 Oct 2017 16:27:43 -0700 Subject: [PATCH 485/573] Add bazel mirror links for newly added workspace dependencies. PiperOrigin-RevId: 173732606 --- tensorflow/workspace.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index c0eb87a744..e25e12d5c5 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -727,6 +727,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "javax_validation", jar_sha256 = "e459f313ebc6db2483f8ceaad39af07086361b474fa92e40f442e8de5d9895dc", jar_urls = [ + "http://mirror.bazel.build/repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar", "http://repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar", ], licenses = ["notice"], # Apache 2.0 @@ -785,6 +786,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5", strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d", urls = [ + "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", ], build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")), @@ -796,6 +798,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): strip_prefix = "flatbuffers-971a68110e4fc1bace10fcb6deeb189e7e1a34ce", sha256 = "874088d2ee0d9f8524191f77209556415f03dd44e156276edf19e5b90ceb5f55", urls = [ + "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz", "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz", ], ) -- GitLab From 3ff9c8d2af2ede9ee7c16fb3b15d004e423f95e5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 16:36:18 -0700 Subject: [PATCH 486/573] Fix typos in Linear Model Tutorial samples 1. test_file_name is undefined (should be test_file.name) 2. train_file_name is undefined (should be train_file.name) PiperOrigin-RevId: 173733442 --- tensorflow/docs_src/tutorials/wide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md index 6292c1a01e..ba16e12a72 100644 --- a/tensorflow/docs_src/tutorials/wide.md +++ b/tensorflow/docs_src/tutorials/wide.md @@ -383,7 +383,7 @@ API: ```python # set num_epochs to None to get infinite stream of data. m.train( - input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True), + input_fn=input_fn(train_file.name, num_epochs=None, shuffle=True), steps=train_steps) ``` @@ -392,7 +392,7 @@ the labels of the holdout data: ```python results = m.evaluate( - input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False), + input_fn=input_fn(test_file.name, num_epochs=1, shuffle=False), steps=None) print("model directory = %s" % model_dir) for key in sorted(results): -- GitLab From d1c59bd37510b9fa1e0cd909c1f4857028d4d13b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 17:03:46 -0700 Subject: [PATCH 487/573] Add tf.quantize op, which is the same as tf.quantize_v2. PiperOrigin-RevId: 173735986 --- tensorflow/core/kernels/quantize_op.cc | 1 - tensorflow/core/ops/array_ops.cc | 12 ++++---- tensorflow/python/ops/array_ops.py | 30 ++++++++++++++++++-- tensorflow/tools/api/golden/tensorflow.pbtxt | 4 +++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/kernels/quantize_op.cc b/tensorflow/core/kernels/quantize_op.cc index 75aa47cd6b..fc26813a08 100644 --- a/tensorflow/core/kernels/quantize_op.cc +++ b/tensorflow/core/kernels/quantize_op.cc @@ -250,5 +250,4 @@ REGISTER_KERNEL_BUILDER( REGISTER_KERNEL_BUILDER( Name("QuantizeV2").Device(DEVICE_CPU).TypeConstraint("T"), QuantizeV2Op); - } // namespace tensorflow diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index f73bc716d5..cdf370399c 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -4902,10 +4902,10 @@ with the range of qint8. If the mode is 'MIN_FIRST', then this approach is used: ``` -number_of_steps = 1 << (# of bits in T) -range_adjust = number_of_steps / (number_of_steps - 1) +num_discrete_values = 1 << (# of bits in T) +range_adjust = num_discrete_values / (num_discrete_values - 1) range = (range_max - range_min) * range_adjust -range_scale = number_of_steps / range +range_scale = num_discrete_values / range quantized = round(input * range_scale) - round(range_min * range_scale) + numeric_limits::min() quantized = max(quantized, numeric_limits::min()) @@ -5017,10 +5017,10 @@ each value by 128 prior to casting. If the mode is 'MIN_FIRST', then this approach is used: ```c++ -number_of_steps = 1 << (# of bits in T) -range_adjust = number_of_steps / (number_of_steps - 1) +num_discrete_values = 1 << (# of bits in T) +range_adjust = num_discrete_values / (num_discrete_values - 1) range = (range_max - range_min) * range_adjust -range_scale = range / number_of_steps +range_scale = range / num_discrete_values const double offset_input = static_cast(input) - lowest_quantized; result = range_min + ((input - numeric_limits::min()) * range_scale) ``` diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index ba8c611f57..8a447deea2 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -66,6 +66,7 @@ See the @{$python/array_ops} guide. @@one_hot @@sequence_mask @@dequantize +@@quantize @@quantize_v2 @@quantized_concat @@setdiff1d @@ -2525,7 +2526,10 @@ gather.__doc__ = gen_array_ops.gather_v2.__doc__ # Define quantize_v2 here in order to make name the second-to-last attribute, # because round_mode was added later. -def quantize_v2(input, +@deprecation.deprecated( + "2017-10-25", + "`tf.quantize_v2` is deprecated, please use `tf.quantize` instead.") +def quantize_v2(input, # pylint: disable=redefined-builtin min_range, max_range, T, @@ -2541,4 +2545,26 @@ def quantize_v2(input, round_mode=round_mode) -quantize_v2.__doc__ = gen_array_ops.quantize_v2.__doc__ +quantize_v2.__doc__ = """Please use `tf.quantize` instead.""" + + +# We want to expose tf.quantize instead of tf.quantize_v2; we can deprecate +# tf.quantize_v2 in next version of TensorFlow. +def quantize(input, # pylint: disable=redefined-builtin + min_range, + max_range, + T, + mode="MIN_COMBINED", + round_mode="HALF_AWAY_FROM_ZERO", + name=None): + return gen_array_ops.quantize_v2( + input, + min_range, + max_range, + T, + mode=mode, + round_mode=round_mode, + name=name) + + +quantize.__doc__ = gen_array_ops.quantize_v2.__doc__ diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index 1c6f3cc534..f61f82e43e 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -1480,6 +1480,10 @@ tf_module { name: "qr" argspec: "args=[\'input\', \'full_matrices\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "quantize" + argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'HALF_AWAY_FROM_ZERO\', \'None\'], " + } member_method { name: "quantize_v2" argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], " -- GitLab From 245a5c171aa7ec4787080b6e0a88f281e1345f97 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Fri, 27 Oct 2017 17:24:33 -0700 Subject: [PATCH 488/573] Make functional_ops compatible with eager exeuction by ignoring caching devices when in eager mode PiperOrigin-RevId: 173737949 --- tensorflow/python/BUILD | 1 + .../kernel_tests/functional_ops_test.py | 100 ++++++++++------- tensorflow/python/ops/functional_ops.py | 101 +++++++++++------- 3 files changed, 128 insertions(+), 74 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 4de5d7f7db..d435ae1375 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -897,6 +897,7 @@ py_library( ":tensor_shape", ":util", ":variable_scope", + "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py index 21fe588ac1..f5717a5a21 100644 --- a/tensorflow/python/kernel_tests/functional_ops_test.py +++ b/tensorflow/python/kernel_tests/functional_ops_test.py @@ -52,6 +52,7 @@ def simple_scoped_fn(a, x): class FunctionalOpsTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testFoldl_Simple(self): with self.test_session(): elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data") @@ -59,13 +60,13 @@ class FunctionalOpsTest(test.TestCase): r = functional_ops.foldl( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems) - self.assertAllEqual(208, r.eval()) + self.assertAllEqual(208, self.evaluate(r)) r = functional_ops.foldl( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems, initializer=10) - self.assertAllEqual(880, r.eval()) + self.assertAllEqual(880, self.evaluate(r)) def testFoldl_Scoped(self): with self.test_session() as sess: @@ -78,14 +79,15 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(variables.trainable_variables()[0].name, "root/body/two:0") sess.run([variables.global_variables_initializer()]) - self.assertAllEqual(208, r.eval()) + self.assertAllEqual(208, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.foldl(simple_scoped_fn, elems, initializer=10) self.assertEqual(len(variables.trainable_variables()), 1) - self.assertAllEqual(880, r.eval()) + self.assertAllEqual(880, self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testFoldr_Simple(self): with self.test_session(): elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data") @@ -93,13 +95,13 @@ class FunctionalOpsTest(test.TestCase): r = functional_ops.foldr( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems) - self.assertAllEqual(450, r.eval()) + self.assertAllEqual(450, self.evaluate(r)) r = functional_ops.foldr( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems, initializer=10) - self.assertAllEqual(1282, r.eval()) + self.assertAllEqual(1282, self.evaluate(r)) def testFoldr_Scoped(self): with self.test_session() as sess: @@ -112,13 +114,13 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(variables.trainable_variables()[0].name, "root/body/two:0") sess.run([variables.global_variables_initializer()]) - self.assertAllEqual(450, r.eval()) + self.assertAllEqual(450, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.foldr(simple_scoped_fn, elems, initializer=10) self.assertEqual(len(variables.trainable_variables()), 1) - self.assertAllEqual(1282, r.eval()) + self.assertAllEqual(1282, self.evaluate(r)) # pylint: disable=unnecessary-lambda def testFold_Grad(self): @@ -128,21 +130,23 @@ class FunctionalOpsTest(test.TestCase): r = functional_ops.foldl( lambda a, x: math_ops.multiply(a, x), elems, initializer=v) r = gradients_impl.gradients(r, v)[0] - self.assertAllEqual(720.0, r.eval()) + self.assertAllEqual(720.0, self.evaluate(r)) r = functional_ops.foldr( lambda a, x: math_ops.multiply(a, x), elems, initializer=v) r = gradients_impl.gradients(r, v)[0] - self.assertAllEqual(720.0, r.eval()) + self.assertAllEqual(720.0, self.evaluate(r)) # pylint: enable=unnecessary-lambda + @test_util.run_in_graph_and_eager_modes() def testMap_Simple(self): with self.test_session(): nums = [1, 2, 3, 4, 5, 6] elems = constant_op.constant(nums, name="data") r = functional_ops.map_fn( lambda x: math_ops.multiply(math_ops.add(x, 3), 2), elems) - self.assertAllEqual(np.array([(x + 3) * 2 for x in nums]), r.eval()) + self.assertAllEqual( + np.array([(x + 3) * 2 for x in nums]), self.evaluate(r)) def testMapSparseTensor(self): with self.test_session(): @@ -177,13 +181,13 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(variables.trainable_variables()[0].name, "root/body/two:0") sess.run([variables.global_variables_initializer()]) - self.assertAllEqual(doubles, r.eval()) + self.assertAllEqual(doubles, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.map_fn(double_scoped, elems) self.assertEqual(len(variables.trainable_variables()), 1) - self.assertAllEqual(doubles, r.eval()) + self.assertAllEqual(doubles, self.evaluate(r)) def testMap_Grad(self): with self.test_session(): @@ -192,19 +196,22 @@ class FunctionalOpsTest(test.TestCase): y = functional_ops.map_fn( lambda x: math_ops.multiply(math_ops.square(x), param), elems) r = gradients_impl.gradients(y, param)[0] - self.assertAllEqual(91.0, r.eval()) + self.assertAllEqual(91.0, self.evaluate(r)) r = gradients_impl.gradients(y, elems)[0] - self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], r.eval()) + self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testMap_SimpleNotTensor(self): with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) r = functional_ops.map_fn( lambda x: math_ops.multiply(math_ops.add(x, 3), 2), nums) - self.assertAllEqual(np.array([(x + 3) * 2 for x in nums]), r.eval()) + self.assertAllEqual( + np.array([(x + 3) * 2 for x in nums]), self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testMap_SingleInputMultiOutput(self): - with self.test_session() as sess: + with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) r = functional_ops.map_fn( lambda x: ((x + 3) * 2, -(x + 3) * 2), @@ -213,10 +220,11 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(2, len(r)) self.assertEqual((6,), r[0].get_shape()) self.assertEqual((6,), r[1].get_shape()) - received = sess.run(r) + received = self.evaluate(r) self.assertAllEqual((nums + 3) * 2, received[0]) self.assertAllEqual(-(nums + 3) * 2, received[1]) + @test_util.run_in_graph_and_eager_modes() def testMap_MultiOutputMismatchedDtype(self): with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) @@ -228,6 +236,7 @@ class FunctionalOpsTest(test.TestCase): nums, dtype=[dtypes.int64, dtypes.int64]) + @test_util.run_in_graph_and_eager_modes() def testMap_MultiInputSingleOutput(self): with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) @@ -235,11 +244,12 @@ class FunctionalOpsTest(test.TestCase): lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)), dtype=dtypes.int64) self.assertEqual((6,), r.get_shape()) - received = r.eval() + received = self.evaluate(r) self.assertAllEqual(nums * nums + (-nums), received) + @test_util.run_in_graph_and_eager_modes() def testMap_MultiInputSameStructureOutput(self): - with self.test_session() as sess: + with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) r = functional_ops.map_fn(lambda x: (x[1][0], (x[1][1], x[0])), (nums, (2 * nums, -nums))) @@ -247,11 +257,12 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual((6,), r[0].get_shape()) self.assertEqual((6,), r[1].get_shape()) self.assertEqual((6,), r[2].get_shape()) - received = sess.run(r) + received = self.evaluate(r) self.assertAllEqual(2 * nums, received[0]) self.assertAllEqual(-nums, received[1]) self.assertAllEqual(nums, received[2]) + @test_util.run_in_graph_and_eager_modes() def testScan_Simple(self): with self.test_session(): elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data") @@ -259,24 +270,26 @@ class FunctionalOpsTest(test.TestCase): # pylint: disable=unnecessary-lambda r = functional_ops.scan(lambda a, x: math_ops.multiply(a, x), elems) - self.assertAllEqual([1., 2., 6., 24., 120., 720.], r.eval()) + self.assertAllEqual([1., 2., 6., 24., 120., 720.], self.evaluate(r)) r = functional_ops.scan( lambda a, x: math_ops.multiply(a, x), elems, initializer=v) - self.assertAllEqual([2., 4., 12., 48., 240., 1440.], r.eval()) + self.assertAllEqual([2., 4., 12., 48., 240., 1440.], self.evaluate(r)) # pylint: enable=unnecessary-lambda + @test_util.run_in_graph_and_eager_modes() def testScan_SingleInputMultiOutput(self): - with self.test_session() as sess: + with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) initializer = (np.array(1.0), np.array(-1.0)) r = functional_ops.scan(lambda a, x: (a[0] * x, -a[1] * x), elems, initializer) - r_value = sess.run(r) + r_value = self.evaluate(r) self.assertAllEqual([1.0, 2.0, 6.0, 24.0, 120.0, 720.0], r_value[0]) self.assertAllEqual([1.0, -2.0, 6.0, -24.0, 120.0, -720.0], r_value[1]) + @test_util.run_in_graph_and_eager_modes() def testScan_MultiInputSingleOutput(self): with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) @@ -284,17 +297,19 @@ class FunctionalOpsTest(test.TestCase): # Multiply a * 1 each time r = functional_ops.scan(lambda a, x: a * (x[0] + x[1]), (elems + 1, -elems), initializer) - self.assertAllEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], r.eval()) + self.assertAllEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testScan_MultiInputSameTypeOutput(self): - with self.test_session() as sess: + with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) r = functional_ops.scan(lambda a, x: (a[0] + x[0], a[1] + x[1]), (elems, -elems)) - r_value = sess.run(r) + r_value = self.evaluate(r) self.assertAllEqual(np.cumsum(elems), r_value[0]) self.assertAllEqual(np.cumsum(-elems), r_value[1]) + @test_util.run_in_graph_and_eager_modes() def testScan_MultiOutputMismatchedInitializer(self): with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) @@ -316,15 +331,16 @@ class FunctionalOpsTest(test.TestCase): "root/body/two:0") sess.run([variables.global_variables_initializer()]) results = np.array([1, 6, 18, 44, 98, 208]) - self.assertAllEqual(results, r.eval()) + self.assertAllEqual(results, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.scan(simple_scoped_fn, elems, initializer=2) self.assertEqual(len(variables.trainable_variables()), 1) results = np.array([6, 16, 38, 84, 178, 368]) - self.assertAllEqual(results, r.eval()) + self.assertAllEqual(results, self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testScanFoldl_Nested(self): with self.test_session(): elems = constant_op.constant([1.0, 2.0, 3.0, 4.0], name="data") @@ -346,7 +362,7 @@ class FunctionalOpsTest(test.TestCase): # t == 3, a == 2.25, x == 4 (returns 9) # t_0 == 0, b == a == 2.25, y == 0.5, returns b * y * x = 4.5 # t_1 == 1, b == 4.5, y == 0.5, returns b * y * x = 9 - self.assertAllClose([1., 1., 2.25, 9.], r.eval()) + self.assertAllClose([1., 1., 2.25, 9.], self.evaluate(r)) def testScan_Control(self): with self.test_session() as sess: @@ -369,7 +385,7 @@ class FunctionalOpsTest(test.TestCase): lambda a, x: math_ops.multiply(a, x), elems, initializer=v) # pylint: enable=unnecessary-lambda r = gradients_impl.gradients(r, v)[0] - self.assertAllEqual(873.0, r.eval()) + self.assertAllEqual(873.0, self.evaluate(r)) def testScanGradientWithPartStopGradient(self): a = variables.Variable(0.0, name="a") @@ -383,6 +399,7 @@ class FunctionalOpsTest(test.TestCase): variables.global_variables_initializer().run() sess.run(grad) + @test_util.run_in_graph_and_eager_modes() def testFoldShape(self): with self.test_session(): x = constant_op.constant([[1, 2, 3], [4, 5, 6]]) @@ -392,32 +409,37 @@ class FunctionalOpsTest(test.TestCase): initializer = constant_op.constant([0, 0, 0]) y = functional_ops.foldl(fn, x, initializer=initializer) - self.assertAllEqual(y.get_shape(), y.eval().shape) + self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) + @test_util.run_in_graph_and_eager_modes() def testMapShape(self): with self.test_session(): x = constant_op.constant([[1, 2, 3], [4, 5, 6]]) y = functional_ops.map_fn(lambda e: e, x) - self.assertAllEqual(y.get_shape(), y.eval().shape) + self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) def testMapUnknownShape(self): x = array_ops.placeholder(dtypes.float32) y = functional_ops.map_fn(lambda e: e, x) self.assertIs(None, y.get_shape().dims) + @test_util.run_in_graph_and_eager_modes() def testMapEmptyScalar(self): with self.test_session(): map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([])) self.assertAllEqual([0], map_return.get_shape().dims) - self.assertAllEqual([0], map_return.eval().shape) + self.assertAllEqual([0], self.evaluate(map_return).shape) + # TODO(akshayka): this test fails in eager: the iterable is of length 0 so + # so the body of the while loop never executes def testMapEmptyTensor(self): with self.test_session(): map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]), constant_op.constant([])) self.assertAllEqual([0, 3, 2], map_return.get_shape().dims) - self.assertAllEqual([0, 3, 2], map_return.eval().shape) + self.assertAllEqual([0, 3, 2], self.evaluate(map_return).shape) + @test_util.run_in_graph_and_eager_modes() def testScanShape(self): with self.test_session(): x = constant_op.constant([[1, 2, 3], [4, 5, 6]]) @@ -427,14 +449,16 @@ class FunctionalOpsTest(test.TestCase): initializer = constant_op.constant([0, 0, 0]) y = functional_ops.scan(fn, x, initializer=initializer) - self.assertAllEqual(y.get_shape(), y.eval().shape) + self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) + # TODO(akshayka): this test fails in eager: the iterable is of length 0 so + # so the body of the while loop never executes def testScanEmptyTensor(self): with self.test_session(): x = functional_ops.scan( lambda x, _: x, math_ops.range(0), initializer=array_ops.ones([2, 4])) self.assertAllEqual([0, 2, 4], x.get_shape()) - self.assertAllEqual(x.get_shape(), x.eval().shape) + self.assertAllEqual(x.get_shape(), self.evaluate(x).shape) def testScanUnknownShape(self): x = array_ops.placeholder(dtypes.float32) diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py index 96b799f610..688512bea6 100644 --- a/tensorflow/python/ops/functional_ops.py +++ b/tensorflow/python/ops/functional_ops.py @@ -27,6 +27,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor @@ -87,15 +88,20 @@ def foldl(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, if not callable(fn): raise TypeError("fn must be callable.") + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "foldl", [elems]): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally + # and not issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True # Convert elems to tensor array. elems = ops.convert_to_tensor(elems, name="elems") @@ -121,7 +127,9 @@ def foldl(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, back_prop=back_prop, swap_memory=swap_memory) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return r_a @@ -167,15 +175,20 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, if not callable(fn): raise TypeError("fn must be callable.") + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "foldr", [elems]): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally and not + # issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True # Convert elems to tensor array. elems = ops.convert_to_tensor(elems, name="elems") @@ -201,7 +214,9 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, back_prop=back_prop, swap_memory=swap_memory) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return r_a @@ -324,15 +339,20 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True, elems_flat = input_flatten(elems) + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "map", elems_flat): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally + # and not issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True elems_flat = [ ops.convert_to_tensor(elem, name="elem") for elem in elems_flat] @@ -396,7 +416,9 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True, r.set_shape(tensor_shape.TensorShape(n_static).concatenate( r.get_shape()[1:])) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return output_pack(results_flat) @@ -509,15 +531,20 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, elems_flat = input_flatten(elems) + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "scan", elems_flat): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally + # and not issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True # Convert elems to tensor array. elems_flat = [ @@ -594,7 +621,9 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, r.set_shape(tensor_shape.TensorShape(n_static).concatenate( r.get_shape()[1:])) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return output_pack(results_flat) -- GitLab From fb2c84cb27c7427455245c20fb22fb2489895b2e Mon Sep 17 00:00:00 2001 From: Jeremy Lau Date: Fri, 27 Oct 2017 17:32:30 -0700 Subject: [PATCH 489/573] Internal change PiperOrigin-RevId: 173738655 --- tensorflow/compiler/tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index d07bf98296..0ff99c5156 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -464,7 +464,7 @@ tf_xla_py_test( tf_xla_py_test( name = "unary_ops_test", - size = "small", + size = "medium", srcs = ["unary_ops_test.py"], deps = [ ":xla_test", -- GitLab From 48df7c97296472730e8547bb3aa59e6730e956cd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 17:34:05 -0700 Subject: [PATCH 490/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173738765 --- tensorflow/core/ops/ops.pbtxt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index e43ee0d986..f41cb212be 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -7049,7 +7049,7 @@ op { } } summary: "Dequantize the \'input\' tensor into a float Tensor." - description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6. The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / number_of_steps\nconst double offset_input = static_cast(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```" + description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6. The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnum_discrete_values = 1 << (# of bits in T)\nrange_adjust = num_discrete_values / (num_discrete_values - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / num_discrete_values\nconst double offset_input = static_cast(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```" } op { name: "DeserializeIterator" @@ -17500,7 +17500,7 @@ op { } } summary: "Quantize the \'input\' tensor of type float to \'output\' tensor of type \'T\'." - description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents. The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n numeric_limits::min()\nquantized = max(quantized, numeric_limits::min())\nquantized = min(quantized, numeric_limits::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them." + description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents. The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnum_discrete_values = 1 << (# of bits in T)\nrange_adjust = num_discrete_values / (num_discrete_values - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = num_discrete_values / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n numeric_limits::min()\nquantized = max(quantized, numeric_limits::min())\nquantized = min(quantized, numeric_limits::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them." } op { name: "QuantizedAdd" -- GitLab From ca56fa49a7755ba2bbd3f586dbaaaefe9e16327d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 17:39:28 -0700 Subject: [PATCH 491/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173739110 --- tensorflow/go/op/wrappers.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 615c386858..ebe4a51116 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -1904,10 +1904,10 @@ func DequantizeMode(value string) DequantizeAttr { // If the mode is 'MIN_FIRST', then this approach is used: // // ```c++ -// number_of_steps = 1 << (# of bits in T) -// range_adjust = number_of_steps / (number_of_steps - 1) +// num_discrete_values = 1 << (# of bits in T) +// range_adjust = num_discrete_values / (num_discrete_values - 1) // range = (range_max - range_min) * range_adjust -// range_scale = range / number_of_steps +// range_scale = range / num_discrete_values // const double offset_input = static_cast(input) - lowest_quantized; // result = range_min + ((input - numeric_limits::min()) * range_scale) // ``` @@ -13766,10 +13766,10 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr { // If the mode is 'MIN_FIRST', then this approach is used: // // ``` -// number_of_steps = 1 << (# of bits in T) -// range_adjust = number_of_steps / (number_of_steps - 1) +// num_discrete_values = 1 << (# of bits in T) +// range_adjust = num_discrete_values / (num_discrete_values - 1) // range = (range_max - range_min) * range_adjust -// range_scale = number_of_steps / range +// range_scale = num_discrete_values / range // quantized = round(input * range_scale) - round(range_min * range_scale) + // numeric_limits::min() // quantized = max(quantized, numeric_limits::min()) -- GitLab From 729db035e7aaa8811dccff154dc582fab12ccee3 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Fri, 27 Oct 2017 17:47:54 -0700 Subject: [PATCH 492/573] Allow compatibility notes in class, property and module doc-strings PiperOrigin-RevId: 173739674 --- tensorflow/tools/docs/pretty_docs.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/docs/pretty_docs.py b/tensorflow/tools/docs/pretty_docs.py index 92f50189dd..c033c16ae9 100644 --- a/tensorflow/tools/docs/pretty_docs.py +++ b/tensorflow/tools/docs/pretty_docs.py @@ -117,7 +117,8 @@ def _build_class_page(page_info): parts.append(page_info.guides) parts.append(page_info.doc.docstring) parts.append(_build_function_details(page_info.doc.function_details)) - assert not page_info.doc.compatibility + parts.append(_build_compatibility(page_info.doc.compatibility)) + parts.append('\n\n') if page_info.classes: @@ -139,7 +140,8 @@ def _build_class_page(page_info): parts.append(prop_info.doc.docstring) parts.append(_build_function_details(prop_info.doc.function_details)) - assert not prop_info.doc.compatibility + parts.append(_build_compatibility(prop_info.doc.compatibility)) + parts.append('\n\n') parts.append('\n\n') @@ -206,6 +208,8 @@ def _build_module_page(page_info): parts.append(str(page_info.defined_in)) parts.append(page_info.doc.docstring) + parts.append(_build_compatibility(page_info.doc.compatibility)) + parts.append('\n\n') if page_info.modules: -- GitLab From 09a89ae57d92b9753c76fa298d373468cb05cc6a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 18:01:37 -0700 Subject: [PATCH 493/573] Add `tf.contrib.distributions.bijectors.Reshape`. PiperOrigin-RevId: 173740491 --- tensorflow/contrib/distributions/BUILD | 16 + .../kernel_tests/bijectors/reshape_test.py | 242 ++++++++++++++ .../python/ops/bijectors/__init__.py | 2 + .../python/ops/bijectors/reshape.py | 29 ++ .../python/ops/bijectors/reshape_impl.py | 297 ++++++++++++++++++ 5 files changed, 586 insertions(+) create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/reshape.py create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index bc72bc37a7..4a4f378901 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -913,6 +913,22 @@ cuda_py_test( ], ) +cuda_py_test( + name = "reshape_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/reshape_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "sigmoid_test", size = "small", diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py new file mode 100644 index 0000000000..38b3a23c2d --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py @@ -0,0 +1,242 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Reshape Bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.distributions.python.ops.bijectors.reshape import Reshape +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import array_ops +from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite +from tensorflow.python.platform import test + + +class ReshapeBijectorTest(test.TestCase): + """Tests correctness of the reshape transformation.""" + + def setUp(self): + self._rng = np.random.RandomState(42) + + def testBijector(self): + """Do a basic sanity check of forward, inverse, jacobian.""" + expected_x = np.random.randn(4, 3, 2) + expected_y = np.reshape(expected_x, [4, 6]) + + with self.test_session() as sess: + bijector = Reshape( + event_shape_out=[6,], + event_shape_in=[3, 2], + validate_args=True) + (x_, + y_, + fldj_, + ildj_) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + bijector.forward_log_det_jacobian(expected_x), + bijector.inverse_log_det_jacobian(expected_y), + )) + self.assertEqual("reshape", bijector.name) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + self.assertAllClose(0., fldj_, rtol=1e-6, atol=0) + self.assertAllClose(0., ildj_, rtol=1e-6, atol=0) + + def testEventShapeDynamicNdims(self): + """Check forward/inverse shape methods with dynamic ndims.""" + + shape_in = tensor_shape.TensorShape([6,]) + shape_in_ph = array_ops.placeholder(dtype=dtypes.int32) + + shape_out = tensor_shape.TensorShape([2, 3]) + shape_out_ph = array_ops.placeholder(dtype=dtypes.int32) + + bijector = Reshape( + event_shape_out=shape_out_ph, + event_shape_in=shape_in_ph, validate_args=True) + + # using the _tensor methods, we should always get a fully-specified + # result since these are evaluated at graph runtime. + with self.test_session() as sess: + (shape_out_, + shape_in_) = sess.run(( + bijector.forward_event_shape_tensor(shape_in), + bijector.inverse_event_shape_tensor(shape_out), + ), feed_dict={ + shape_in_ph: shape_in, + shape_out_ph: shape_out, + }) + self.assertAllEqual(shape_out, shape_out_) + self.assertAllEqual(shape_in, shape_in_) + + def testEventShapeDynamic(self): + """Check shape methods with static ndims but dynamic shape.""" + + shape_in = tensor_shape.TensorShape([6,]) + shape_in_partial = tensor_shape.TensorShape([None,]) + shape_in_ph = array_ops.placeholder( + shape=[1,], dtype=dtypes.int32) + + shape_out = tensor_shape.TensorShape([2, 3]) + shape_out_partial = tensor_shape.TensorShape([None, None]) + shape_out_ph = array_ops.placeholder( + shape=[2,], dtype=dtypes.int32) + + bijector = Reshape( + event_shape_out=shape_out_ph, + event_shape_in=shape_in_ph, + validate_args=True) + + # if event shapes are not statically available, should + # return partially-specified TensorShapes. + self.assertAllEqual( + bijector.forward_event_shape(shape_in).as_list(), + shape_out_partial.as_list()) + self.assertAllEqual( + bijector.inverse_event_shape(shape_out).as_list(), + shape_in_partial.as_list()) + + # using the _tensor methods, we should always get a fully-specified + # result since these are evaluated at graph runtime. + with self.test_session() as sess: + (shape_out_, + shape_in_) = sess.run(( + bijector.forward_event_shape_tensor(shape_in), + bijector.inverse_event_shape_tensor(shape_out), + ), feed_dict={ + shape_in_ph: shape_in, + shape_out_ph: shape_out, + }) + self.assertAllEqual(shape_out, shape_out_) + self.assertAllEqual(shape_in, shape_in_) + + def testEventShapeStatic(self): + """Check shape methods when shape is statically known.""" + + shape_in = tensor_shape.TensorShape([6,]) + shape_out = tensor_shape.TensorShape([2, 3]) + + bijector_static = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + + # test that forward_ and inverse_event_shape do sensible things + # when shapes are statically known. + self.assertEqual( + bijector_static.forward_event_shape(shape_in), + shape_out) + self.assertEqual( + bijector_static.inverse_event_shape(shape_out), + shape_in) + + with self.test_session() as sess: + (shape_out_static_, + shape_in_static_, + ) = sess.run(( + bijector_static.forward_event_shape_tensor(shape_in), + bijector_static.inverse_event_shape_tensor(shape_out), + )) + self.assertAllEqual(shape_out, shape_out_static_) + self.assertAllEqual(shape_in, shape_in_static_) + + def testScalarReshape(self): + """Test reshaping to and from a scalar shape ().""" + + expected_x = np.random.randn(4, 3, 1) + expected_y = np.reshape(expected_x, [4, 3]) + + expected_x_scalar = np.random.randn(1,) + expected_y_scalar = expected_x_scalar[0] + + with self.test_session() as sess: + bijector = Reshape( + event_shape_out=[], + event_shape_in=[1,], validate_args=True) + + (x_, + y_, + x_scalar_, + y_scalar_ + ) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + bijector.inverse(expected_y_scalar), + bijector.forward(expected_x_scalar), + )) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + self.assertAllClose(expected_y_scalar, y_scalar_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x_scalar, x_scalar_, rtol=1e-6, atol=0) + + def testRaisesOpError(self): + x1 = np.random.randn(4, 2, 3) + x2 = np.random.randn(4, 3, 2) + x3 = np.random.randn(4, 5, 1, 1) + + with self.test_session() as sess: + shape_in_ph = array_ops.placeholder(shape=[2,], dtype=dtypes.int32) + shape_out_ph = array_ops.placeholder(shape=[3,], dtype=dtypes.int32) + bijector = Reshape( + event_shape_out=shape_out_ph, + event_shape_in=shape_in_ph, + validate_args=True) + + with self.assertRaisesOpError( + "Input `event_shape` does not match `event_shape_in`."): + sess.run(bijector.forward(x2), + feed_dict={shape_out_ph: [1, 6, 1], + shape_in_ph: [2, 3]}) + + with self.assertRaisesOpError( + "event_shape_out entries must be positive."): + sess.run(bijector.forward(x1), + feed_dict={shape_out_ph: [-1, -1, 6], + shape_in_ph: [2, 3]}) + + # test that *all* methods check basic assertions + fd_mismatched = {shape_out_ph: [1, 1, 5], shape_in_ph: [2, 3]} + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.forward(x1), feed_dict=fd_mismatched) + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.inverse(x3), feed_dict=fd_mismatched) + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.inverse_log_det_jacobian(x3), + feed_dict=fd_mismatched) + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.forward_log_det_jacobian(x1), + feed_dict=fd_mismatched) + + def testBijectiveAndFinite(self): + x = np.random.randn(4, 2, 3) + y = np.reshape(x, [4, 1, 2, 3]) + with self.test_session(): + bijector = Reshape( + event_shape_in=[2, 3], + event_shape_out=[1, 2, 3], + validate_args=True) + assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py index fd6c509446..bc0ec7f195 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py @@ -29,6 +29,7 @@ @@MaskedAutoregressiveFlow @@Permute @@PowerTransform +@@Reshape @@Sigmoid @@SigmoidCentered @@SinhArcsinh @@ -59,6 +60,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.invert import * from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import * from tensorflow.contrib.distributions.python.ops.bijectors.permute import * from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import * +from tensorflow.contrib.distributions.python.ops.bijectors.reshape import * from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import * from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid_centered import * from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import * diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py new file mode 100644 index 0000000000..8997f7ab69 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py @@ -0,0 +1,29 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reshape bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.contrib.distributions.python.ops.bijectors.reshape_impl import * +# pylint: enable=wildcard-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = ["Reshape"] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py new file mode 100644 index 0000000000..93682639aa --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py @@ -0,0 +1,297 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reshape bijectors.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bijector as bijector_lib + + +__all__ = [ + "Reshape", +] + + +class Reshape(bijector_lib.Bijector): + """Reshapes the `event_shape` of a `Tensor`. + + The semantics generally follow that of `tf.reshape()`, with + a few differences: + * The user must provide both the input and output shape, so that + the transformation can be inverted. + * The `Reshape` bijector automatically broadcasts over the leftmost + dimensions of its input (`sample_shape` and `batch_shape`); only + the rightmost `event_ndims_in` dimensions are reshaped. The + number of dimensions to reshape is inferred from the provided + `event_shape_in` (`event_ndims_in = len(event_shape_in)`). + * The `Reshape` bijector does not currently support + partially-specified shapes, i.e., those with a dimension + implicitly specified by `-1`. + + Example usage: + ```python + + bs = tf.contrib.distributions.bijectors + + reverse = bs.Reshape(event_shape_out=[1,2], + event_shape_in=[2,]) + + reverse.forward([1., 2.]) # shape [2,] + # ==> [[1., 2.]] # shape [1,2] + + reverse.forward([[1., 2.], [3., 4.]]) # shape [2, 2] + # ==> [[[1., 2.]], [[3., 4.]]] # shape [2, 1, 2] + + reverse.inverse([[1., 2.]]) # shape [1,2] + # ==> [1., 2.] # shape [2,] + + reverse.forward_log_det_jacobian(any_value) + # ==> 0. + + reverse.inverse_log_det_jacobian(any_value) + # ==> 0. + ``` + + """ + + def __init__(self, event_shape_out, event_shape_in, + validate_args=False, name=None): + """Creates a `Reshape` bijector. + + Args: + event_shape_out: An `int`-like vector-shaped `Tensor` + representing the fully specified (no -1's) event shape of the + transformed output. + event_shape_in: An `int`-like vector-shaped `Tensor` + representing the fully specified (no -1's) event shape of the + input. + validate_args: Python `bool` indicating whether arguments should + be checked for correctness. + name: Python `str`, name given to ops managed by this object. + + Raises: + TypeError: if either `event_shape_in` or `event_shape_out` has + non-vector shape (`rank > 1`), or non-integer `dtype`. + ValueError: if either `event_shape_in` or `event_shape_out` + contains non-positive entries, or if their sizes do not match + (`prod(event_shape_in)` != `prod(event_shape_out)`), or if + their dimensionality(s) cannot be statically inferred. + """ + with ops.name_scope(name, "reshape", + values=[event_shape_out, event_shape_in]): + + event_shape_out = ops.convert_to_tensor(event_shape_out, + name="event_shape_out", + preferred_dtype=dtypes.int32) + event_shape_in = ops.convert_to_tensor(event_shape_in, + name="event_shape_in", + preferred_dtype=dtypes.int32) + + # check that input shapes are positive integers + assertions = [] + assertions += self._maybe_check_valid_shape( + event_shape_out, "event_shape_out", + validate_args=validate_args) + assertions += self._maybe_check_valid_shape( + event_shape_in, "event_shape_in", validate_args=validate_args) + + # check that prod(event_shape_in) = prod(event_shape_out) + assertions += self._maybe_check_matching_sizes( + event_shape_in, event_shape_out, validate_args=validate_args) + + self._assertions = assertions + self._event_shape_in = event_shape_in + self._event_shape_out = event_shape_out + self._event_shape_in_static = tensor_util.constant_value_as_shape( + event_shape_in) + self._event_shape_out_static = tensor_util.constant_value_as_shape( + event_shape_out) + + super(Reshape, self).__init__(is_constant_jacobian=True, + validate_args=validate_args, + name=name or "reshape") + + def _maybe_check_valid_shape(self, shape_tensor, label, + validate_args=False): + """Check that a shape Tensor is int-type and positive.""" + + assertions = [] + + if not shape_tensor.dtype.is_integer: + raise TypeError("{} dtype ({}) should be `int`-like.".format( + label, shape_tensor.dtype.name)) + + shape_rank = tensor_util.constant_value(array_ops.rank(shape_tensor)) + if shape_rank is not None and shape_rank > 1: + raise ValueError("{} rank should be <= 1.".format(label)) + + s = tensor_util.constant_value(shape_tensor) + if s is not None: + if (s <= 0).any(): + raise ValueError("{} entries must be positive, but found {}".format( + label, s)) + elif validate_args: + assertions.append(check_ops.assert_positive( + shape_tensor, message="{} entries must be positive".format(label))) + + return assertions + + def _maybe_check_matching_sizes(self, event_shape_in, event_shape_out, + validate_args=False): + """Check that prod(event_shape_in)==prod(event_shape_out).""" + + def _get_size_from_shape(shape): + """Computes size from a shape `Tensor`, statically if possible.""" + s = tensor_util.constant_value(shape) + if s is not None: + return [np.int32(np.prod(s))]*2 + return None, math_ops.reduce_prod(shape, name="size") + + # Ensure `event_shape_in` is compatible with `event_shape_out`. + event_size_in_, event_size_in = _get_size_from_shape( # pylint: disable=unbalanced-tuple-unpacking + event_shape_in) + event_size_out_, event_size_out = _get_size_from_shape( # pylint: disable=unbalanced-tuple-unpacking + event_shape_out) + + assertions = [] + if event_size_in_ is not None and event_size_out_ is not None: + if event_size_in_ != event_size_out_: + raise ValueError( + "Input `event_size` ({}) does not match output `event_size` ({}).". + format(event_size_in, event_size_out_)) + elif validate_args: + assertions.append(check_ops.assert_equal( + event_size_in, event_size_out, + message="Input/output `event_size`s do not match.")) + + return assertions + + def _reshape_helper(self, x, event_shape_in, event_shape_out): + """Reshape only the event_shape of an input `Tensor`.""" + + def _get_rank_from_shape(shape): + """Computes rank from a shape `Tensor`, statically if possible.""" + # Uses fact that rank is "shape of shape". + ndims = shape.shape.with_rank_at_least(1)[0].value + if ndims is not None: + return ndims, ndims + return None, array_ops.shape(shape)[0] + + event_ndims_in_, event_ndims_in = _get_rank_from_shape(event_shape_in) + + assertions = [] + # Ensure x.event_shape is compatible with event_shape_in. + if x.shape.ndims is not None: + x_ndims_, x_ndims = [x.shape.ndims]*2 + else: + x_ndims_, x_ndims = None, array_ops.rank(x) + + if (event_ndims_in_ is not None + and x_ndims_ is not None + and x.shape.with_rank_at_least(event_ndims_in_)[ + x_ndims_-event_ndims_in_:].is_fully_defined()): + x_event_shape_, x_event_shape = [ # pylint: disable=unbalanced-tuple-unpacking + np.int32(x.shape[x_ndims_-event_ndims_in_:])]*2 + else: + x_event_shape_, x_event_shape = ( + None, array_ops.shape(x)[x_ndims-event_ndims_in:]) + + event_shape_in_ = tensor_util.constant_value(event_shape_in) + + if x_event_shape_ is not None and event_shape_in_ is not None: + if not np.equal(x_event_shape_, event_shape_in_).all(): + raise ValueError( + "Input `event_shape` ({}) does not match `event_shape_in` ({}).". + format(x_event_shape_, event_shape_in_)) + elif self.validate_args: + assertions.append(check_ops.assert_equal( + x_event_shape, event_shape_in, + message="Input `event_shape` does not match `event_shape_in`.")) + + if assertions: + x = control_flow_ops.with_dependencies(assertions, x) + + # get the parts of shape(x) that will not change + sample_and_batch_shape = array_ops.shape(x) + + ndims = (x.shape.ndims if x.shape.ndims is not None + else array_ops.rank(x)) + sample_and_batch_shape = sample_and_batch_shape[ + :(ndims - math_ops.abs(event_ndims_in))] + + new_shape = array_ops.concat( + [sample_and_batch_shape, event_shape_out], axis=0) + + return array_ops.reshape(x, new_shape) + + def _forward(self, x): + with ops.control_dependencies(self._assertions): + return self._reshape_helper(x, + self._event_shape_in, + self._event_shape_out) + + def _inverse(self, y): + with ops.control_dependencies(self._assertions): + return self._reshape_helper(y, + self._event_shape_out, + self._event_shape_in) + + def _inverse_log_det_jacobian(self, y): + with ops.control_dependencies(self._assertions): + return constant_op.constant(0., dtype=y.dtype) + + def _forward_log_det_jacobian(self, x): + with ops.control_dependencies(self._assertions): + return constant_op.constant(0., dtype=x.dtype) + + def _forward_event_shape(self, input_shape): + self._event_shape_in_static.assert_is_compatible_with(input_shape) + return self._event_shape_out_static + + def _inverse_event_shape(self, output_shape): + self._event_shape_out_static.assert_is_compatible_with(output_shape) + return self._event_shape_in_static + + def _forward_event_shape_tensor(self, input_shape): + input_assertions = self._maybe_check_valid_shape( + input_shape, "input event shape", validate_args=self.validate_args) + input_assertions += self._maybe_check_matching_sizes( + input_shape, self._event_shape_out, + validate_args=self.validate_args) + + return control_flow_ops.with_dependencies( + input_assertions + self._assertions, self._event_shape_out) + + def _inverse_event_shape_tensor(self, output_shape): + + output_assertions = self._maybe_check_valid_shape( + output_shape, "output event shape", validate_args=self.validate_args) + output_assertions += self._maybe_check_matching_sizes( + output_shape, self._event_shape_in, validate_args=self.validate_args) + + return control_flow_ops.with_dependencies( + output_assertions + self._assertions, self._event_shape_in) -- GitLab From 45c5118f0e924c1b2212dc97ad535c35891c66b0 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Fri, 27 Oct 2017 18:11:01 -0700 Subject: [PATCH 494/573] When creating an HloModule from an HloProto construct the HloModuleConfig with a correct ProgramShape which matches the shapes of the entry computation. Previously the module config had a bogus or default constructed ProgramShape. PiperOrigin-RevId: 173741104 --- tensorflow/compiler/xla/service/hlo_module.cc | 113 +++++++++++++++++- tensorflow/compiler/xla/service/hlo_module.h | 11 +- tensorflow/compiler/xla/service/hlo_runner.cc | 9 +- 3 files changed, 121 insertions(+), 12 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 4779ec7760..1758f2760c 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -204,13 +204,93 @@ HloModuleProto HloModule::ToProto() const { return proto; } +namespace { + +// Construct a ProgramShape matching the shape of the parameters and root of the +// given module's entry computation. +StatusOr ProgramShapeFromProto(const HloModuleProto& module) { + const HloComputationProto* entry_computation = nullptr; + for (const HloComputationProto& computation : module.computations()) { + if (computation.name() == module.entry_computation_name()) { + entry_computation = &computation; + break; + } + } + TF_RET_CHECK(entry_computation != nullptr) + << "No computation with entry computation name" + << module.entry_computation_name(); + + tensorflow::gtl::FlatMap> parameters; + const HloInstructionProto* root = nullptr; + for (const HloInstructionProto& instruction : + entry_computation->instructions()) { + if (instruction.name() == entry_computation->root_name()) { + TF_RET_CHECK(root == nullptr) << "Entry computation has more than " + "one instruction with (root) name " + << instruction.name(); + root = &instruction; + } + if (instruction.opcode() == HloOpcodeString(HloOpcode::kParameter)) { + TF_RET_CHECK(!ContainsKey(parameters, instruction.parameter_number())) + << "Entry computation has more than one parameter instruction " + "with parameter number " + << instruction.parameter_number(); + parameters[instruction.parameter_number()] = { + instruction.parameter_name(), &instruction.shape()}; + } + } + TF_RET_CHECK(root != nullptr) + << "Entry computation is missing root instruction named " + << entry_computation->root_name(); + + ProgramShape program_shape; + *program_shape.mutable_result() = root->shape(); + for (int64 i = 0; i < parameters.size(); ++i) { + TF_RET_CHECK(ContainsKey(parameters, i)) + << "Entry computation missing parameter number " << i; + const string& name = parameters.at(i).first; + const Shape& shape = *parameters.at(i).second; + *program_shape.add_parameters() = shape; + program_shape.add_parameter_names(name); + } + + return std::move(program_shape); +} + +} // namespace + /* static */ StatusOr> HloModule::CreateFromProto( - const HloModuleProto& proto, - const VersionedComputationHandle& entry_computation_handle, - const HloModuleConfig& config) { - auto module = - MakeUnique(proto.name(), entry_computation_handle, config); + const HloModuleProto& proto, const HloModuleConfig& module_config, + const VersionedComputationHandle& entry_computation_handle) { + // The ProgramShape in the passed in module config must match the shapes of + // the entry parameters and root. + TF_ASSIGN_OR_RETURN(ProgramShape expected_program_shape, + ProgramShapeFromProto(proto)); + TF_RET_CHECK(expected_program_shape.parameters_size() == + module_config.entry_computation_layout().parameter_count()); + for (int i = 0; i < expected_program_shape.parameters_size(); ++i) { + const Shape& parameter_shape = + module_config.entry_computation_layout().parameter_layout(i).shape(); + TF_RET_CHECK( + ShapeUtil::Equal(expected_program_shape.parameters(i), parameter_shape)) + << "HloModuleConfig has different shape for parameter " << i + << " than the HLO module. Expected: " + << ShapeUtil::HumanStringWithLayout( + expected_program_shape.parameters(i)) + << ", actual: " << ShapeUtil::HumanStringWithLayout(parameter_shape); + } + const Shape& result_shape = + module_config.entry_computation_layout().result_layout().shape(); + TF_RET_CHECK(ShapeUtil::Equal(expected_program_shape.result(), result_shape)) + << "HloModuleConfig has different result shape than the HLO module. " + "Expected: " + << ShapeUtil::HumanStringWithLayout(expected_program_shape.result()) + << ", actual: " << ShapeUtil::HumanStringWithLayout(result_shape); + + auto module = MakeUnique(proto.name(), entry_computation_handle, + module_config); + tensorflow::gtl::FlatMap computation_map; for (const HloComputationProto& computation_proto : proto.computations()) { TF_ASSIGN_OR_RETURN(std::unique_ptr computation, @@ -250,6 +330,29 @@ StatusOr> HloModule::CreateFromProto( return std::move(module); } +/* static */ +StatusOr HloModule::CreateModuleConfigFromProto( + const HloModuleProto& module) { + TF_ASSIGN_OR_RETURN(ProgramShape program_shape, + ProgramShapeFromProto(module)); + + HloModuleConfig module_config(program_shape); + + // The module config is constructed with default layouts regardless of what is + // passed in via the ProgramShape. Set the layouts to the appropriate values. + ComputationLayout* entry_layout = + module_config.mutable_entry_computation_layout(); + for (int64 i = 0; i < entry_layout->parameter_count(); ++i) { + TF_RETURN_IF_ERROR( + entry_layout->mutable_parameter_layout(i)->CopyLayoutFromShape( + program_shape.parameters(i))); + } + TF_RETURN_IF_ERROR(entry_layout->mutable_result_layout()->CopyLayoutFromShape( + program_shape.result())); + + return module_config; +} + namespace { // Returns whether `hlo` is used outside the given subcomputation. // `instructions_in_subcomputation` is the instruction set of the given diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h index 2ac4244e5c..ad11d56006 100644 --- a/tensorflow/compiler/xla/service/hlo_module.h +++ b/tensorflow/compiler/xla/service/hlo_module.h @@ -144,9 +144,14 @@ class HloModule { // Convert an HloModule to or from a proto. HloModuleProto ToProto() const; static StatusOr> CreateFromProto( - const HloModuleProto& proto, - const VersionedComputationHandle& entry_computation_handle, - const HloModuleConfig& config); + const HloModuleProto& proto, const HloModuleConfig& module_config, + const VersionedComputationHandle& entry_computation_handle = + VersionedComputationHandle()); + + // Creates and returns an HloModuleConfig with an appropriate program shape + // for the HLO module in the given proto. + static StatusOr CreateModuleConfigFromProto( + const HloModuleProto& module); // Outlines the given expression from the given computation. // instructions_to_outline contains the instructions that form the expression. diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index 9fdda38d2d..c3f74e253f 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -45,11 +45,12 @@ HloRunner::ReadModuleFromHloProtoFile(const char* filename, HloProto proto; TF_RETURN_IF_ERROR(tensorflow::ReadBinaryProto(tensorflow::Env::Default(), filename, &proto)); - HloModuleConfig config; + TF_ASSIGN_OR_RETURN( + HloModuleConfig config, + HloModule::CreateModuleConfigFromProto(proto.hlo_module())); config.set_debug_options(debug_options); - TF_ASSIGN_OR_RETURN(auto module, HloModule::CreateFromProto( - proto.hlo_module(), - VersionedComputationHandle(), config)); + TF_ASSIGN_OR_RETURN(auto module, + HloModule::CreateFromProto(proto.hlo_module(), config)); return std::move(module); } -- GitLab From 9f4b12bb55d102988ad9c3c064e37d85b1c4e38e Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 27 Oct 2017 18:16:08 -0700 Subject: [PATCH 495/573] [XLA] DOT dumper: Print constant shape when we elide the constant's value. For example, instead of "operand 1 = %constant.42", we now print "operand 1 = %constant.42 (f32[100])". PiperOrigin-RevId: 173741373 --- tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index d0202556bc..ed23c8c2dd 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -761,10 +761,14 @@ string HloDotDumper::GetInstructionNodeInlinedOperands( return Printf("%s (%s)", constant->literal().GetAsString(elem_idx), ShapeUtil::HumanString(constant->shape())); } + string constant_name; if (tensorflow::StringPiece(constant->name()).starts_with("%constant")) { - return constant->name(); + constant_name = constant->name(); + } else { + constant_name = StrCat("constant ", constant->name()); } - return StrCat("constant ", constant->name()); + return Printf("%s %s", constant_name, + ShapeUtil::HumanString(constant->shape())); }; // Special case: If instr is a parameter to a fusion node, check whether the -- GitLab From 36696ad58305cf5bc654c86dd8d6db881154b438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E4=BC=A0=E6=AD=A6?= Date: Fri, 27 Oct 2017 02:08:44 -0500 Subject: [PATCH 496/573] tf.zeros doesn't accept a tensor argument ValueError: Shape must be rank 1 but is rank 0 for 'zeros_2' (op: 'Fill') with input shapes: [], []. --- tensorflow/docs_src/programmers_guide/tensors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/tensors.md b/tensorflow/docs_src/programmers_guide/tensors.md index cc4181e75e..d6f80430cd 100644 --- a/tensorflow/docs_src/programmers_guide/tensors.md +++ b/tensorflow/docs_src/programmers_guide/tensors.md @@ -197,7 +197,7 @@ For example, here is how to make a vector of zeros with the same size as the number of columns in a given matrix: ``` python -zeros = tf.zeros(tf.shape(my_matrix)[1]) +zeros = tf.zeros(my_matrix.shape[1]) ``` ### Changing the shape of a `tf.Tensor` -- GitLab From e8a62a30b35153e3ba8d32bdfd5845e1f92fe46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=94=B0=E4=BC=A0=E6=AD=A6?= Date: Fri, 27 Oct 2017 20:47:08 -0500 Subject: [PATCH 497/573] Fix minor typo --- tensorflow/docs_src/programmers_guide/graphs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 5ec3738d7d..c08043835a 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -89,7 +89,7 @@ to all API functions in the same context. For example: * Executing `v = tf.Variable(0)` adds to the graph a @{tf.Operation} that will store a writeable tensor value that persists between @{tf.Session.run} calls. The @{tf.Variable} object wraps this operation, and can be used [like a - tensor](#tensor-like-objects), which will read the current value of the + tensor](#tensor-like_objects), which will read the current value of the stored value. The @{tf.Variable} object also has methods such as @{tf.Variable.assign$`assign`} and @{tf.Variable.assign_add$`assign_add`} that create @{tf.Operation} objects that, when executed, update the stored value. -- GitLab From c16797ec365fcfa730ea6a3ffc6a4c227056fcd8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 27 Oct 2017 19:26:27 -0700 Subject: [PATCH 498/573] Adds eager execution compatibility note in Estimators. Raises a RuntimeError in Estimator base class. PiperOrigin-RevId: 173744765 --- tensorflow/python/estimator/canned/dnn.py | 8 ++++++++ .../python/estimator/canned/dnn_linear_combined.py | 8 ++++++++ tensorflow/python/estimator/canned/linear.py | 8 ++++++++ tensorflow/python/estimator/estimator.py | 11 ++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/estimator/canned/dnn.py b/tensorflow/python/estimator/canned/dnn.py index a3e3756007..8e90fd4ec6 100644 --- a/tensorflow/python/estimator/canned/dnn.py +++ b/tensorflow/python/estimator/canned/dnn.py @@ -259,6 +259,10 @@ class DNNClassifier(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, @@ -392,6 +396,10 @@ class DNNRegressor(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, diff --git a/tensorflow/python/estimator/canned/dnn_linear_combined.py b/tensorflow/python/estimator/canned/dnn_linear_combined.py index ff4ecee5c0..3c61bd5b07 100644 --- a/tensorflow/python/estimator/canned/dnn_linear_combined.py +++ b/tensorflow/python/estimator/canned/dnn_linear_combined.py @@ -278,6 +278,10 @@ class DNNLinearCombinedClassifier(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, @@ -438,6 +442,10 @@ class DNNLinearCombinedRegressor(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, diff --git a/tensorflow/python/estimator/canned/linear.py b/tensorflow/python/estimator/canned/linear.py index 3338f8ee2c..8658ee38e9 100644 --- a/tensorflow/python/estimator/canned/linear.py +++ b/tensorflow/python/estimator/canned/linear.py @@ -184,6 +184,10 @@ class LinearClassifier(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, @@ -300,6 +304,10 @@ class LinearRegressor(estimator.Estimator): key=column.name, value=a `Tensor` Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index f198b051cf..6243cfc118 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -29,6 +29,7 @@ import six from tensorflow.core.framework import summary_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as tf_session +from tensorflow.python.eager import context from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator import run_config from tensorflow.python.estimator import util @@ -87,6 +88,10 @@ class Estimator(object): None of `Estimator`'s methods can be overridden in subclasses (its constructor enforces this). Subclasses should use `model_fn` to configure the base class, and may add methods implementing specialized functionality. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, model_fn, model_dir=None, config=None, params=None): @@ -129,10 +134,15 @@ class Estimator(object): Keys are names of parameters, values are basic python types. Raises: + RuntimeError: If eager execution is enabled. ValueError: parameters of `model_fn` don't match `params`. ValueError: if this is called via a subclass and if that class overrides a member of `Estimator`. """ + if context.in_eager_mode(): + raise RuntimeError( + 'Estimators are not supported when eager execution is enabled.') + Estimator._assert_members_are_not_overridden(self) if config is None: @@ -1016,4 +1026,3 @@ def _has_dataset_or_queue_runner(maybe_tensor): # Now, check queue. return ops.get_default_graph().get_collection(ops.GraphKeys.QUEUE_RUNNERS) - -- GitLab From 8ec7540e008f37abc9fb7c0bb02dd0b25c4b7b78 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 27 Oct 2017 21:24:28 -0700 Subject: [PATCH 499/573] TFE: Fix pip test for tf.contrib.summary Fixes test failure in tensorflow/contrib/summary:summary_ops_test, e.g., http://ci.tensorflow.org/job/tensorflow-cl-cpu-python3-pip/10933/console PiperOrigin-RevId: 173749502 --- tensorflow/tools/pip_package/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 579c51ab3a..cba8e89209 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -167,6 +167,7 @@ sh_binary( "//tensorflow/contrib/slim/python/slim/data:data_pip", "//tensorflow/contrib/slim/python/slim/nets:nets_pip", "//tensorflow/contrib/specs:specs", + "//tensorflow/contrib/summary:summary_test_util", "//tensorflow/contrib/tensor_forest:init_py", "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip", "//tensorflow/contrib/timeseries:timeseries_pip", -- GitLab From e7645b629568c3ef968fa0dddeb2ff01a67e55e2 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 27 Oct 2017 22:43:46 -0700 Subject: [PATCH 500/573] [XLA] DOT dumper: Handle fusion nodes nested inside other nodes (e.g. map). PiperOrigin-RevId: 173752314 --- .../compiler/xla/service/hlo_graph_dumper.cc | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index ed23c8c2dd..e000a06706 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -360,6 +360,21 @@ class HloDotDumper { string GetInstructionNodeInlinedOperands(const HloInstruction* instr); void AddInstructionIncomingEdges(const HloInstruction* instr); + // For most instructions, GetNodeForEdge(instr) returns instr. + // + // The exception is fusion nodes. For these, we walk up the chain of nested + // fusion nodes starting at instr until we reach a node that either (a) isn't + // a fusion node, or (b) is a fusion node for which + // ShouldShowFusionSubcomputation is false. + // + // We do this because fusion nodes are expanded inline -- if + // ShouldShowFusionSubcomputation is true, the fusion node won't be present in + // the graph. + // + // In general when you want to draw an edge from A to B, you should actually + // draw an edge from GetNodeForEdge(A) to GetNodeForEdge(B). + const HloInstruction* GetNodeForEdge(const HloInstruction* instr); + // If instr has just one computation and it's trivial (e.g. "return param0 + // param1"), returns a string you can put into the node's body that names the // subcomputation, e.g. "Subcomputation: add". @@ -595,16 +610,15 @@ tooltip = " "; // belongs to a fusion node, it's drawn in place of the fusion instruction, // so there's no need to link those. if (parent_instr->opcode() != HloOpcode::kFusion) { - VLOG(2) << "Edge: from " << subcomp->root_instruction()->name() << " to " - << parent_instr->name() << " as " << next_edge_id_; - edge_ids_.insert( - {{subcomp->root_instruction(), parent_instr}, next_edge_id_++}); + const HloInstruction* from = GetNodeForEdge(subcomp->root_instruction()); + VLOG(2) << "Edge: from " << from->name() << " to " << parent_instr->name() + << " as " << next_edge_id_; + edge_ids_.insert({{from, parent_instr}, next_edge_id_++}); const char* edge_fmt = R"(%s -> %s [ltail="%s", style="dashed" tooltip="%s -> %s"];)"; - edges_.push_back( - Printf(edge_fmt, InstructionId(subcomp->root_instruction()), - InstructionId(parent_instr), SubcomputationId(subcomp), - subcomp->name(), parent_instr->name())); + edges_.push_back(Printf( + edge_fmt, InstructionId(from), InstructionId(parent_instr), + SubcomputationId(subcomp), subcomp->name(), parent_instr->name())); } string computation = @@ -633,15 +647,7 @@ string HloDotDumper::DumpComputation(const HloComputation* comp) { } string HloDotDumper::DumpRootTag() { - HloInstruction* from = computation_->root_instruction(); - - // Fusion nodes are expanded inline, so if root is an expanded fusion node, - // walk up the graph until we find a node that isn't. - while (from->opcode() == HloOpcode::kFusion && - ShouldShowFusionSubcomputation(from)) { - from = from->fused_expression_root(); - } - + const HloInstruction* from = GetNodeForEdge(computation_->root_instruction()); auto from_id = InstructionId(from); if (!filter_.Show(from)) { @@ -1080,13 +1086,8 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { void HloDotDumper::AddInstructionIncomingEdges(const HloInstruction* instr) { auto add_edge = [&](const HloInstruction* from, const HloInstruction* to, int64 operand_num, bool control_edge = false) { - // Fusion nodes' subcomputations are displayed inline, so if 'from' is a - // fusion node and the node's subcomputation is shown, we draw our edge - // starting at the fusion node's root instead of at the fusion node itself. - if (from->opcode() == HloOpcode::kFusion && - ShouldShowFusionSubcomputation(from)) { - from = from->fused_expression_root(); - } + from = GetNodeForEdge(from); + if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant || ShouldMergeIntoUsers(from)) { return; @@ -1154,6 +1155,15 @@ string HloDotDumper::GetInstructionTrivialComputationStr( return Join(lines, "
"); } +const HloInstruction* HloDotDumper::GetNodeForEdge( + const HloInstruction* instr) { + while (instr->opcode() == HloOpcode::kFusion && + ShouldShowFusionSubcomputation(instr)) { + instr = instr->fused_expression_root(); + } + return instr; +} + tensorflow::mutex& RendererMutex() { static tensorflow::mutex* mu = new tensorflow::mutex; return *mu; -- GitLab From 0eba15fe6349ae2bd50b14496a1f283f462b0c66 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 28 Oct 2017 08:30:04 -0700 Subject: [PATCH 501/573] Adds eager compatability message for PartitionedVariable. PiperOrigin-RevId: 173772851 --- tensorflow/python/ops/variables.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index fd0aee3c33..187aa5d8e0 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -1053,7 +1053,16 @@ class Variable(object): class PartitionedVariable(object): - """A container for partitioned `Variable` objects.""" + """A container for partitioned `Variable` objects. + + @compatiblity(eager) `tf.PartitionedVariable` is not compatible with + eager execution. Use `tfe.Variable` instead which is compatable + with both eager execution and graph construction. See [the + TensorFlow Eager Execution + guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) + for details on how variables work in eager execution. + @end_compatiblity + """ class PartitionedVariableIterator(object): """An iterator that allows accessing the underlying `Variable` objects. @@ -1102,10 +1111,11 @@ class PartitionedVariable(object): `partitions` is not a list. ValueError: If `variable_list` is empty, or the `Variable` shape information does not match `shape`, or `partitions` has invalid values. - RuntimeError: If created in EAGER mode. + RuntimeError: If eager execution is enabled """ if not context.in_graph_mode(): - raise RuntimeError("PartitionedVariable not supported in Eager mode.") + raise RuntimeError("tf.PartitionedVariable not supported in " + "eager mode. Please use tfe.Variable instead") if not isinstance(variable_list, (list, tuple)): raise TypeError( "variable_list is not a list or tuple: %s" % variable_list) -- GitLab From 325c8e5efa003bdc53f9605eb0b272075abc3565 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sat, 28 Oct 2017 23:45:55 -0700 Subject: [PATCH 502/573] Improve C++ SQLite veneer - Use shared_ptr for Sqlite - Don't need unique_ptr on SqliteStatement - Don't need db namespace - Include SQL in error statuses PiperOrigin-RevId: 173802267 --- .../contrib/cmake/tf_core_framework.cmake | 4 + tensorflow/contrib/tensorboard/db/schema.cc | 13 +- tensorflow/contrib/tensorboard/db/schema.h | 6 +- .../contrib/tensorboard/db/schema_test.cc | 7 +- .../kernels/sql/sqlite_query_connection.cc | 34 +-- .../kernels/sql/sqlite_query_connection.h | 4 +- tensorflow/core/lib/db/BUILD | 1 + tensorflow/core/lib/db/sqlite.cc | 50 +++-- tensorflow/core/lib/db/sqlite.h | 61 ++++- tensorflow/core/lib/db/sqlite_test.cc | 209 +++++++++++------- 10 files changed, 244 insertions(+), 145 deletions(-) diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index 1b64a52ece..c3dc8531bb 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -191,6 +191,10 @@ file(GLOB_RECURSE tf_core_lib_srcs "${tensorflow_source_dir}/tensorflow/core/lib/*.h" "${tensorflow_source_dir}/tensorflow/core/lib/*.cc" "${tensorflow_source_dir}/tensorflow/core/public/*.h" + # TODO(@jart): Move StatusOr into core. + "${tensorflow_source_dir}/tensorflow/compiler/xla/statusor.cc" + "${tensorflow_source_dir}/tensorflow/compiler/xla/statusor.h" + "${tensorflow_source_dir}/tensorflow/compiler/xla/statusor_internals.h" ) file(GLOB tf_core_platform_srcs diff --git a/tensorflow/contrib/tensorboard/db/schema.cc b/tensorflow/contrib/tensorboard/db/schema.cc index f5a8e02a9b..98fff9e0ae 100644 --- a/tensorflow/contrib/tensorboard/db/schema.cc +++ b/tensorflow/contrib/tensorboard/db/schema.cc @@ -15,13 +15,11 @@ limitations under the License. #include "tensorflow/contrib/tensorboard/db/schema.h" namespace tensorflow { -namespace db { namespace { class SqliteSchema { public: - explicit SqliteSchema(Sqlite* db) : db_(db) {} - ~SqliteSchema() { db_ = nullptr; } + explicit SqliteSchema(std::shared_ptr db) : db_(std::move(db)) {} /// \brief Creates Tensors table. /// @@ -371,18 +369,18 @@ class SqliteSchema { Status Run(const char* sql) { auto stmt = db_->Prepare(sql); - TF_RETURN_WITH_CONTEXT_IF_ERROR(stmt->StepAndReset(), sql); + TF_RETURN_WITH_CONTEXT_IF_ERROR(stmt.StepAndReset(), sql); return Status::OK(); } private: - Sqlite* db_; + std::shared_ptr db_; }; } // namespace -Status SetupTensorboardSqliteDb(Sqlite* db) { - SqliteSchema s(db); +Status SetupTensorboardSqliteDb(std::shared_ptr db) { + SqliteSchema s(std::move(db)); TF_RETURN_IF_ERROR(s.CreateTensorsTable()); TF_RETURN_IF_ERROR(s.CreateTensorChunksTable()); TF_RETURN_IF_ERROR(s.CreateTagsTable()); @@ -408,5 +406,4 @@ Status SetupTensorboardSqliteDb(Sqlite* db) { return Status::OK(); } -} // namespace db } // namespace tensorflow diff --git a/tensorflow/contrib/tensorboard/db/schema.h b/tensorflow/contrib/tensorboard/db/schema.h index d3a6922d94..900c10298c 100644 --- a/tensorflow/contrib/tensorboard/db/schema.h +++ b/tensorflow/contrib/tensorboard/db/schema.h @@ -15,19 +15,19 @@ limitations under the License. #ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ #define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ +#include + #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/db/sqlite.h" namespace tensorflow { -namespace db { /// \brief Creates TensorBoard SQLite tables and indexes. /// /// If they are already created, this has no effect. If schema /// migrations are necessary, they will be performed with logging. -Status SetupTensorboardSqliteDb(Sqlite* db); +Status SetupTensorboardSqliteDb(std::shared_ptr db); -} // namespace db } // namespace tensorflow #endif // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ diff --git a/tensorflow/contrib/tensorboard/db/schema_test.cc b/tensorflow/contrib/tensorboard/db/schema_test.cc index a4302dda44..463c4e59e7 100644 --- a/tensorflow/contrib/tensorboard/db/schema_test.cc +++ b/tensorflow/contrib/tensorboard/db/schema_test.cc @@ -20,15 +20,12 @@ limitations under the License. #include "tensorflow/core/platform/test.h" namespace tensorflow { -namespace db { namespace { TEST(SchemaTest, SmokeTestTensorboardSchema) { - std::unique_ptr db; - TF_ASSERT_OK(Sqlite::Open(":memory:", &db)); - TF_ASSERT_OK(SetupTensorboardSqliteDb(db.get())); + auto db = Sqlite::Open(":memory:").ValueOrDie(); + TF_ASSERT_OK(SetupTensorboardSqliteDb(db)); } } // namespace -} // namespace db } // namespace tensorflow diff --git a/tensorflow/core/kernels/sql/sqlite_query_connection.cc b/tensorflow/core/kernels/sql/sqlite_query_connection.cc index a9e6ee0969..1330506d28 100644 --- a/tensorflow/core/kernels/sql/sqlite_query_connection.cc +++ b/tensorflow/core/kernels/sql/sqlite_query_connection.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #include "tensorflow/core/kernels/sql/sqlite_query_connection.h" + #include "tensorflow/core/lib/strings/stringprintf.h" namespace tensorflow { @@ -29,17 +30,18 @@ Status SqliteQueryConnection::Open(const string& data_source_name, return errors::FailedPrecondition( "Failed to open query connection: Connection already opeend."); } - Status s = db::Sqlite::Open(data_source_name, &db_); + auto s = Sqlite::Open(data_source_name); if (s.ok()) { + db_ = std::move(s.ValueOrDie()); query_ = query; output_types_ = output_types; } - return s; + return s.status(); } Status SqliteQueryConnection::Close() { Status s; - s.Update(stmt_->Close()); + s.Update(stmt_.Close()); s.Update(db_->Close()); return s; } @@ -52,7 +54,7 @@ Status SqliteQueryConnection::GetNext(std::vector* out_tensors, return s; } } - Status s = stmt_->Step(end_of_sequence); + Status s = stmt_.Step(end_of_sequence); if (!*end_of_sequence) { for (int i = 0; i < column_count_; i++) { DataType dt = output_types_[i]; @@ -66,9 +68,9 @@ Status SqliteQueryConnection::GetNext(std::vector* out_tensors, Status SqliteQueryConnection::PrepareQuery() { stmt_ = db_->Prepare(query_); - Status s = stmt_->status(); + Status s = stmt_.status(); if (s.ok()) { - int column_count = stmt_->ColumnCount(); + int column_count = stmt_.ColumnCount(); if (column_count != output_types_.size()) { return errors::InvalidArgument(tensorflow::strings::Printf( "The number of columns in query (%d) must match the number of " @@ -84,40 +86,40 @@ void SqliteQueryConnection::FillTensorWithResultSetEntry( const DataType& data_type, int column_index, Tensor* tensor) { switch (data_type) { case DT_STRING: - tensor->scalar()() = stmt_->ColumnString(column_index); + tensor->scalar()() = stmt_.ColumnString(column_index); break; case DT_INT8: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_INT16: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_INT32: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_INT64: - tensor->scalar()() = stmt_->ColumnInt(column_index); + tensor->scalar()() = stmt_.ColumnInt(column_index); break; case DT_UINT8: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_UINT16: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_BOOL: - tensor->scalar()() = stmt_->ColumnInt(column_index) != 0; + tensor->scalar()() = stmt_.ColumnInt(column_index) != 0; break; case DT_FLOAT: tensor->scalar()() = - static_cast(stmt_->ColumnDouble(column_index)); + static_cast(stmt_.ColumnDouble(column_index)); break; case DT_DOUBLE: - tensor->scalar()() = stmt_->ColumnDouble(column_index); + tensor->scalar()() = stmt_.ColumnDouble(column_index); break; // Error preemptively thrown by SqlDatasetOp::MakeDataset in this case. default: { diff --git a/tensorflow/core/kernels/sql/sqlite_query_connection.h b/tensorflow/core/kernels/sql/sqlite_query_connection.h index b0b4737a1e..435dd8e234 100644 --- a/tensorflow/core/kernels/sql/sqlite_query_connection.h +++ b/tensorflow/core/kernels/sql/sqlite_query_connection.h @@ -42,8 +42,8 @@ class SqliteQueryConnection : public QueryConnection { // `stmt_`. void FillTensorWithResultSetEntry(const DataType& data_type, int column_index, Tensor* tensor); - std::unique_ptr db_ = nullptr; - std::unique_ptr stmt_ = nullptr; + std::shared_ptr db_ = nullptr; + SqliteStatement stmt_; int column_count_ = 0; string query_; DataTypeVector output_types_; diff --git a/tensorflow/core/lib/db/BUILD b/tensorflow/core/lib/db/BUILD index 367686c16a..41b7af1b69 100644 --- a/tensorflow/core/lib/db/BUILD +++ b/tensorflow/core/lib/db/BUILD @@ -12,6 +12,7 @@ cc_library( srcs = ["sqlite.cc"], hdrs = ["sqlite.h"], deps = [ + "//tensorflow/compiler/xla:statusor", "//tensorflow/core:lib", "@sqlite_archive//:sqlite", ], diff --git a/tensorflow/core/lib/db/sqlite.cc b/tensorflow/core/lib/db/sqlite.cc index 108be452a2..701655f622 100644 --- a/tensorflow/core/lib/db/sqlite.cc +++ b/tensorflow/core/lib/db/sqlite.cc @@ -18,14 +18,13 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" namespace tensorflow { -namespace db { /* static */ -Status Sqlite::Open(const string& uri, std::unique_ptr* db) { +xla::StatusOr> Sqlite::Open(const string& uri) { sqlite3* sqlite = nullptr; Status s = MakeStatus(sqlite3_open(uri.c_str(), &sqlite)); if (s.ok()) { - *db = std::unique_ptr(new Sqlite(sqlite)); + return std::shared_ptr(new Sqlite(sqlite)); } return s; } @@ -87,6 +86,9 @@ Sqlite::~Sqlite() { } Status Sqlite::Close() { + if (db_ == nullptr) { + return Status::OK(); + } // If Close is explicitly called, ordering must be correct. Status s = MakeStatus(sqlite3_close(db_)); if (s.ok()) { @@ -95,23 +97,42 @@ Status Sqlite::Close() { return s; } -std::unique_ptr Sqlite::Prepare(const string& sql) { +SqliteStatement Sqlite::Prepare(const string& sql) { sqlite3_stmt* stmt = nullptr; int rc = sqlite3_prepare_v2(db_, sql.c_str(), sql.size() + 1, &stmt, nullptr); - return std::unique_ptr(new SqliteStatement(stmt, rc)); + if (rc == SQLITE_OK) { + return {stmt, SQLITE_OK, std::unique_ptr(nullptr)}; + } else { + return {nullptr, rc, std::unique_ptr(new string(sql))}; + } } -SqliteStatement::SqliteStatement(sqlite3_stmt* stmt, int error) - : stmt_(stmt), error_(error) {} +Status SqliteStatement::status() const { + Status s = Sqlite::MakeStatus(error_); + if (!s.ok()) { + if (stmt_ != nullptr) { + errors::AppendToMessage(&s, sqlite3_sql(stmt_)); + } else { + errors::AppendToMessage(&s, *prepare_error_sql_); + } + } + return s; +} -SqliteStatement::~SqliteStatement() { - int rc = sqlite3_finalize(stmt_); - if (rc != SQLITE_OK) { - LOG(ERROR) << "destruct sqlite3_stmt: " << Sqlite::MakeStatus(rc); +void SqliteStatement::CloseOrLog() { + if (stmt_ != nullptr) { + int rc = sqlite3_finalize(stmt_); + if (rc != SQLITE_OK) { + LOG(ERROR) << "destruct sqlite3_stmt: " << Sqlite::MakeStatus(rc); + } + stmt_ = nullptr; } } Status SqliteStatement::Close() { + if (stmt_ == nullptr) { + return Status::OK(); + } int rc = sqlite3_finalize(stmt_); if (rc == SQLITE_OK) { stmt_ = nullptr; @@ -121,8 +142,10 @@ Status SqliteStatement::Close() { } void SqliteStatement::Reset() { - sqlite3_reset(stmt_); - sqlite3_clear_bindings(stmt_); + if (TF_PREDICT_TRUE(stmt_ != nullptr)) { + sqlite3_reset(stmt_); + sqlite3_clear_bindings(stmt_); // not nullptr friendly + } error_ = SQLITE_OK; } @@ -163,5 +186,4 @@ Status SqliteStatement::StepAndReset() { return s; } -} // namespace db } // namespace tensorflow diff --git a/tensorflow/core/lib/db/sqlite.h b/tensorflow/core/lib/db/sqlite.h index 316e938f1b..774852efea 100644 --- a/tensorflow/core/lib/db/sqlite.h +++ b/tensorflow/core/lib/db/sqlite.h @@ -17,15 +17,16 @@ limitations under the License. #include #include +#include #include "sqlite3.h" +#include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { -namespace db { class SqliteStatement; @@ -46,7 +47,7 @@ class Sqlite { /// `file::memory:` for testing. /// /// See https://sqlite.org/c3ref/open.html - static Status Open(const string& uri, std::unique_ptr* db); + static xla::StatusOr> Open(const string& uri); /// \brief Makes tensorflow::Status for SQLite result code. /// @@ -65,7 +66,7 @@ class Sqlite { /// \brief Frees underlying SQLite object. /// /// Unlike the destructor, all SqliteStatement objects must be closed - /// beforehand. + /// beforehand. This is a no-op if already closed Status Close(); /// \brief Creates SQLite statement. @@ -74,7 +75,7 @@ class Sqlite { /// failed. It is also possible to punt the error checking to after /// the values have been binded and Step() or ExecuteWriteQuery() is /// called. - std::unique_ptr Prepare(const string& sql); + SqliteStatement Prepare(const string& sql); private: explicit Sqlite(sqlite3* db); @@ -89,21 +90,34 @@ class Sqlite { /// Instances of this class are not thread safe. class SqliteStatement { public: - /// \brief Destroys object and finalizes statement if needed. - ~SqliteStatement(); + /// \brief Constructs empty statement that should be assigned later. + SqliteStatement() : stmt_(nullptr), error_(SQLITE_OK) {} + + /// \brief Empties object and finalizes statement if needed. + ~SqliteStatement() { CloseOrLog(); } + + /// \brief Move constructor, after which should not be used. + SqliteStatement(SqliteStatement&& other); + + /// \brief Move assignment, after which should not be used. + SqliteStatement& operator=(SqliteStatement&& other); + + /// \brief Returns true if statement is not empty. + operator bool() const { return stmt_ != nullptr; } /// \brief Returns SQLite result code state. /// /// This will be SQLITE_OK unless an error happened. If multiple /// errors happened, only the first error code will be returned. - int error() { return error_; } + int error() const { return error_; } /// \brief Returns error() as a tensorflow::Status. - Status status() { return Sqlite::MakeStatus(error_); } + Status status() const; /// \brief Finalize statement object. /// - /// Please note that the destructor can also do this. + /// Please note that the destructor can also do this. This method is + /// a no-op if already closed. Status Close(); /// \brief Executes query and/or fetches next row. @@ -247,7 +261,12 @@ class SqliteStatement { private: friend Sqlite; - SqliteStatement(sqlite3_stmt* stmt, int error); // takes ownership + SqliteStatement(sqlite3_stmt* stmt, int error, + std::unique_ptr prepare_error_sql) + : stmt_(stmt), + error_(error), + prepare_error_sql_(std::move(prepare_error_sql)) {} + void CloseOrLog(); void Update(int rc) { if (TF_PREDICT_FALSE(rc != SQLITE_OK)) { @@ -268,11 +287,31 @@ class SqliteStatement { sqlite3_stmt* stmt_; int error_; + std::unique_ptr prepare_error_sql_; TF_DISALLOW_COPY_AND_ASSIGN(SqliteStatement); }; -} // namespace db +inline SqliteStatement::SqliteStatement(SqliteStatement&& other) + : stmt_(other.stmt_), + error_(other.error_), + prepare_error_sql_(std::move(other.prepare_error_sql_)) { + other.stmt_ = nullptr; + other.error_ = SQLITE_OK; +} + +inline SqliteStatement& SqliteStatement::operator=(SqliteStatement&& other) { + if (&other != this) { + CloseOrLog(); + stmt_ = other.stmt_; + error_ = other.error_; + prepare_error_sql_ = std::move(other.prepare_error_sql_); + other.stmt_ = nullptr; + other.error_ = SQLITE_OK; + } + return *this; +} + } // namespace tensorflow #endif // TENSORFLOW_CORE_LIB_DB_SQLITE_H_ diff --git a/tensorflow/core/lib/db/sqlite_test.cc b/tensorflow/core/lib/db/sqlite_test.cc index ce22379d97..ba045274ad 100644 --- a/tensorflow/core/lib/db/sqlite_test.cc +++ b/tensorflow/core/lib/db/sqlite_test.cc @@ -24,97 +24,96 @@ limitations under the License. #include "tensorflow/core/platform/test.h" namespace tensorflow { -namespace db { namespace { class SqliteTest : public ::testing::Test { protected: void SetUp() override { - TF_ASSERT_OK(Sqlite::Open(":memory:", &db_)); + db_ = Sqlite::Open(":memory:").ValueOrDie(); auto stmt = db_->Prepare("CREATE TABLE T (a BLOB, b BLOB)"); - TF_ASSERT_OK(stmt->StepAndReset()); + TF_ASSERT_OK(stmt.StepAndReset()); } - std::unique_ptr db_; + std::shared_ptr db_; bool is_done_; }; TEST_F(SqliteTest, InsertAndSelectInt) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindInt(1, 3); - stmt->BindInt(2, -7); - TF_ASSERT_OK(stmt->StepAndReset()); - stmt->BindInt(1, 123); - stmt->BindInt(2, -123); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindInt(1, 3); + stmt.BindInt(2, -7); + TF_ASSERT_OK(stmt.StepAndReset()); + stmt.BindInt(1, 123); + stmt.BindInt(2, -123); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T ORDER BY b"); - TF_ASSERT_OK(stmt->Step(&is_done_)); + TF_ASSERT_OK(stmt.Step(&is_done_)); ASSERT_FALSE(is_done_); - EXPECT_EQ(123, stmt->ColumnInt(0)); - EXPECT_EQ(-123, stmt->ColumnInt(1)); - TF_ASSERT_OK(stmt->Step(&is_done_)); + EXPECT_EQ(123, stmt.ColumnInt(0)); + EXPECT_EQ(-123, stmt.ColumnInt(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); ASSERT_FALSE(is_done_); - EXPECT_EQ(3, stmt->ColumnInt(0)); - EXPECT_EQ(-7, stmt->ColumnInt(1)); - TF_ASSERT_OK(stmt->Step(&is_done_)); + EXPECT_EQ(3, stmt.ColumnInt(0)); + EXPECT_EQ(-7, stmt.ColumnInt(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); ASSERT_TRUE(is_done_); } TEST_F(SqliteTest, InsertAndSelectDouble) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindDouble(1, 6.28318530); - stmt->BindDouble(2, 1.61803399); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindDouble(1, 6.28318530); + stmt.BindDouble(2, 1.61803399); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(6.28318530, stmt->ColumnDouble(0)); - EXPECT_EQ(1.61803399, stmt->ColumnDouble(1)); - EXPECT_EQ(6, stmt->ColumnInt(0)); - EXPECT_EQ(1, stmt->ColumnInt(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(6.28318530, stmt.ColumnDouble(0)); + EXPECT_EQ(1.61803399, stmt.ColumnDouble(1)); + EXPECT_EQ(6, stmt.ColumnInt(0)); + EXPECT_EQ(1, stmt.ColumnInt(1)); } TEST_F(SqliteTest, NulCharsInString) { string s; // XXX: Want to write {2, '\0'} but not sure why not. s.append(static_cast(2), '\0'); auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlob(1, s); - stmt->BindText(2, s); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindBlob(1, s); + stmt.BindText(2, s); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(2, stmt->ColumnSize(0)); - EXPECT_EQ(2, stmt->ColumnString(0).size()); - EXPECT_EQ('\0', stmt->ColumnString(0).at(0)); - EXPECT_EQ('\0', stmt->ColumnString(0).at(1)); - EXPECT_EQ(2, stmt->ColumnSize(1)); - EXPECT_EQ(2, stmt->ColumnString(1).size()); - EXPECT_EQ('\0', stmt->ColumnString(1).at(0)); - EXPECT_EQ('\0', stmt->ColumnString(1).at(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(2, stmt.ColumnSize(0)); + EXPECT_EQ(2, stmt.ColumnString(0).size()); + EXPECT_EQ('\0', stmt.ColumnString(0).at(0)); + EXPECT_EQ('\0', stmt.ColumnString(0).at(1)); + EXPECT_EQ(2, stmt.ColumnSize(1)); + EXPECT_EQ(2, stmt.ColumnString(1).size()); + EXPECT_EQ('\0', stmt.ColumnString(1).at(0)); + EXPECT_EQ('\0', stmt.ColumnString(1).at(1)); } TEST_F(SqliteTest, Unicode) { string s = "要依法治国是赞美那些谁是公义的和惩罚恶人。 - 韩非"; auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlob(1, s); - stmt->BindText(2, s); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindBlob(1, s); + stmt.BindText(2, s); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(s, stmt->ColumnString(0)); - EXPECT_EQ(s, stmt->ColumnString(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(s, stmt.ColumnString(0)); + EXPECT_EQ(s, stmt.ColumnString(1)); } TEST_F(SqliteTest, StepAndResetClearsBindings) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindInt(1, 1); - stmt->BindInt(2, 123); - TF_ASSERT_OK(stmt->StepAndReset()); - stmt->BindInt(1, 2); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindInt(1, 1); + stmt.BindInt(2, 123); + TF_ASSERT_OK(stmt.StepAndReset()); + stmt.BindInt(1, 2); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT b FROM T ORDER BY a"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(123, stmt->ColumnInt(0)); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(SQLITE_NULL, stmt->ColumnType(0)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(123, stmt.ColumnInt(0)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(SQLITE_NULL, stmt.ColumnType(0)); } TEST_F(SqliteTest, CloseBeforeFinalizeFails) { @@ -128,71 +127,109 @@ TEST_F(SqliteTest, CloseBeforeFinalizeFails) { // is designed to carry the first error state forward to Step(). TEST_F(SqliteTest, ErrorPuntingDoesNotReportLibraryAbuse) { auto stmt = db_->Prepare("lol cat"); - EXPECT_FALSE(stmt->status().ok()); - EXPECT_EQ(SQLITE_ERROR, stmt->error()); - stmt->BindInt(1, 1); - stmt->BindInt(2, 2); - Status s = stmt->Step(&is_done_); - EXPECT_EQ(SQLITE_ERROR, stmt->error()); // first error of several + EXPECT_FALSE(stmt.status().ok()); + EXPECT_EQ(SQLITE_ERROR, stmt.error()); + stmt.BindInt(1, 1); + stmt.BindInt(2, 2); + Status s = stmt.Step(&is_done_); + EXPECT_EQ(SQLITE_ERROR, stmt.error()); // first error of several EXPECT_FALSE(s.ok()); } TEST_F(SqliteTest, SafeBind) { string s = "hello"; auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlob(1, s); - stmt->BindText(2, s); + stmt.BindBlob(1, s); + stmt.BindText(2, s); s.at(0) = 'y'; - TF_ASSERT_OK(stmt->StepAndReset()); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ("hello", stmt->ColumnString(0)); - EXPECT_EQ("hello", stmt->ColumnString(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ("hello", stmt.ColumnString(0)); + EXPECT_EQ("hello", stmt.ColumnString(1)); } TEST_F(SqliteTest, UnsafeBind) { string s = "hello"; auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlobUnsafe(1, s); - stmt->BindTextUnsafe(2, s); + stmt.BindBlobUnsafe(1, s); + stmt.BindTextUnsafe(2, s); s.at(0) = 'y'; - TF_ASSERT_OK(stmt->StepAndReset()); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ("yello", stmt->ColumnString(0)); - EXPECT_EQ("yello", stmt->ColumnString(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ("yello", stmt.ColumnString(0)); + EXPECT_EQ("yello", stmt.ColumnString(1)); } TEST_F(SqliteTest, UnsafeColumn) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindInt(1, 1); - stmt->BindText(2, "hello"); - TF_ASSERT_OK(stmt->StepAndReset()); - stmt->BindInt(1, 2); - stmt->BindText(2, "there"); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindInt(1, 1); + stmt.BindText(2, "hello"); + TF_ASSERT_OK(stmt.StepAndReset()); + stmt.BindInt(1, 2); + stmt.BindText(2, "there"); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT b FROM T ORDER BY a"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - const char* p = stmt->ColumnStringUnsafe(0); + TF_ASSERT_OK(stmt.Step(&is_done_)); + const char* p = stmt.ColumnStringUnsafe(0); EXPECT_EQ('h', *p); - TF_ASSERT_OK(stmt->Step(&is_done_)); + TF_ASSERT_OK(stmt.Step(&is_done_)); // This will actually happen, but it's not safe to test this behavior. // EXPECT_EQ('t', *p); } TEST_F(SqliteTest, NamedParameterBind) { auto stmt = db_->Prepare("INSERT INTO T (a) VALUES (:a)"); - stmt->BindText(":a", "lol"); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindText(":a", "lol"); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT COUNT(*) FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(1, stmt->ColumnInt(0)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(1, stmt.ColumnInt(0)); stmt = db_->Prepare("SELECT a FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); + TF_ASSERT_OK(stmt.Step(&is_done_)); EXPECT_FALSE(is_done_); - EXPECT_EQ("lol", stmt->ColumnString(0)); + EXPECT_EQ("lol", stmt.ColumnString(0)); +} + +TEST_F(SqliteTest, Statement_DefaultConstructor) { + SqliteStatement stmt; + EXPECT_FALSE(stmt); + EXPECT_FALSE(stmt.StepAndReset().ok()); + stmt = db_->Prepare("INSERT INTO T (a) VALUES (1)"); + EXPECT_TRUE(stmt); + EXPECT_TRUE(stmt.StepAndReset().ok()); +} + +TEST_F(SqliteTest, Statement_MoveConstructor) { + SqliteStatement stmt{db_->Prepare("INSERT INTO T (a) VALUES (1)")}; + EXPECT_TRUE(stmt.StepAndReset().ok()); +} + +TEST_F(SqliteTest, Statement_MoveAssignment) { + SqliteStatement stmt1 = db_->Prepare("INSERT INTO T (a) VALUES (1)"); + SqliteStatement stmt2; + EXPECT_TRUE(stmt1.StepAndReset().ok()); + EXPECT_FALSE(stmt2.StepAndReset().ok()); + stmt2 = std::move(stmt1); + EXPECT_TRUE(stmt2.StepAndReset().ok()); +} + +TEST_F(SqliteTest, PrepareFailed) { + SqliteStatement s = db_->Prepare("SELECT"); + EXPECT_FALSE(s.status().ok()); + EXPECT_NE(string::npos, s.status().error_message().find("SELECT")); +} + +TEST_F(SqliteTest, BindFailed) { + SqliteStatement s = db_->Prepare("INSERT INTO T (a) VALUES (123)"); + EXPECT_TRUE(s.status().ok()); + EXPECT_EQ("", s.status().error_message()); + s.BindInt(1, 123); + EXPECT_FALSE(s.status().ok()); + EXPECT_NE(string::npos, + s.status().error_message().find("INSERT INTO T (a) VALUES (123)")); } } // namespace -} // namespace db } // namespace tensorflow -- GitLab From 32ab30cb0a6bc86a6423c9078cfdddac79d79451 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 07:02:33 -0700 Subject: [PATCH 503/573] Fixes typo in compatibility. PiperOrigin-RevId: 173887031 --- tensorflow/python/ops/variables.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 187aa5d8e0..fdd0666403 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -119,13 +119,13 @@ class Variable(object): various `Optimizer` classes use this collection as the default list of variables to optimize. - @compatiblity(eager) + @compatibility(eager) `tf.Variable` is not compatible with eager execution. Use - `tfe.Variable` instead which is compatable with both eager execution + `tfe.Variable` instead which is compatible with both eager execution and graph construction. See [the TensorFlow Eager Execution guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) for details on how variables work in eager execution. - @end_compatiblity + @end_compatibility """ def __init__(self, -- GitLab From 2e54fd6de78d84af6b26f537ee25c5a625adce3b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 07:12:07 -0700 Subject: [PATCH 504/573] Adds eager execution compatibility note in Readers, Queues, and QueueRunner. Raises a RuntimeError in base classes for QueueBase, ReaderBase, and QueueRunner. PiperOrigin-RevId: 173888425 --- tensorflow/python/ops/data_flow_ops.py | 33 ++++++++++++-- tensorflow/python/ops/io_ops.py | 44 +++++++++++++++++++ .../python/training/queue_runner_impl.py | 11 +++++ 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index 62845a9f8b..c186eb5b7e 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -123,6 +123,11 @@ class QueueBase(object): @{tf.RandomShuffleQueue} for concrete implementations of this class, and instructions on how to create them. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, dtypes, shapes, names, queue_ref): @@ -146,12 +151,12 @@ class QueueBase(object): Raises: ValueError: If one of the arguments is invalid. - ValueError: If eager execution is enabled. + RuntimeError: If eager execution is enabled. """ if context.in_eager_mode(): - raise ValueError( - "Queues are not supported in TensorFlow with eager execution. " - "Instead, use tf.data to get data into your model.") + raise RuntimeError( + "Queues are not supported when eager execution is enabled. " + "Instead, please use tf.data to get data into your model.") self._dtypes = dtypes if shapes is not None: if len(shapes) != len(dtypes): @@ -595,6 +600,11 @@ class RandomShuffleQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, min_after_dequeue, dtypes, shapes=None, @@ -668,6 +678,11 @@ class FIFOQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, dtypes, shapes=None, names=None, @@ -719,6 +734,11 @@ class PaddingFIFOQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, dtypes, shapes, names=None, shared_name=None, @@ -781,6 +801,11 @@ class PriorityQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, types, shapes=None, names=None, shared_name=None, diff --git a/tensorflow/python/ops/io_ops.py b/tensorflow/python/ops/io_ops.py index bd879ac423..670bb9a9c2 100644 --- a/tensorflow/python/ops/io_ops.py +++ b/tensorflow/python/ops/io_ops.py @@ -70,6 +70,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.lib.io import python_io @@ -152,6 +153,11 @@ class ReaderBase(object): contains the work units and the Reader dequeues from the queue when it is asked to produce a record (via Read()) but it has finished the last work unit. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, reader_ref, supports_serialize=False): @@ -161,7 +167,15 @@ class ReaderBase(object): reader_ref: The operation that implements the reader. supports_serialize: True if the reader implementation can serialize its state. + + Raises: + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "Readers are not supported when eager execution is enabled. " + "Instead, please use tf.data to get data into your model.") + self._reader_ref = reader_ref self._supports_serialize = supports_serialize @@ -347,6 +361,11 @@ class WholeFileReader(ReaderBase): be a filename (key) and the contents of that file (value). See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, name=None): @@ -367,6 +386,11 @@ class TextLineReader(ReaderBase): Newlines are stripped from the output. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ # TODO(josh11b): Support serializing and restoring state. @@ -390,6 +414,11 @@ class FixedLengthRecordReader(ReaderBase): """A Reader that outputs fixed-length records from a file. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ # TODO(josh11b): Support serializing and restoring state. @@ -427,6 +456,11 @@ class TFRecordReader(ReaderBase): """A Reader that outputs the records from a TFRecords file. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ # TODO(josh11b): Support serializing and restoring state. @@ -452,6 +486,11 @@ class LMDBReader(ReaderBase): """A Reader that outputs the records from a LMDB file. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, name=None, options=None): """Create a LMDBReader. @@ -474,6 +513,11 @@ class IdentityReader(ReaderBase): work string and output (work, work). See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, name=None): diff --git a/tensorflow/python/training/queue_runner_impl.py b/tensorflow/python/training/queue_runner_impl.py index d3b473ee46..4e7c81d7b2 100644 --- a/tensorflow/python/training/queue_runner_impl.py +++ b/tensorflow/python/training/queue_runner_impl.py @@ -44,6 +44,11 @@ class QueueRunner(object): and reporting exceptions, etc. The `QueueRunner`, combined with the `Coordinator`, helps handle these issues. + + @compatibility(eager) + QueueRunners are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, queue=None, enqueue_ops=None, close_op=None, @@ -80,7 +85,13 @@ class QueueRunner(object): ValueError: If both `queue_runner_def` and `queue` are both specified. ValueError: If `queue` or `enqueue_ops` are not provided when not restoring from `queue_runner_def`. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "QueueRunners are not supported when eager execution is enabled. " + "Instead, please use tf.data to get data into your model.") + if queue_runner_def: if queue or enqueue_ops: raise ValueError("queue_runner_def and queue are mutually exclusive.") -- GitLab From ef4490f637e17f3ce599f55522e63d06f470e540 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 07:22:44 -0700 Subject: [PATCH 505/573] BUILD cleanup in contrib/... PiperOrigin-RevId: 173889798 --- tensorflow/contrib/all_reduce/BUILD | 5 +++ tensorflow/contrib/batching/BUILD | 9 +++--- tensorflow/contrib/cloud/BUILD | 4 +++ tensorflow/contrib/cluster_resolver/BUILD | 20 ++++++------ tensorflow/contrib/cudnn_rnn/BUILD | 14 +++++++++ tensorflow/contrib/eager/python/BUILD | 11 ++++--- tensorflow/contrib/estimator/BUILD | 1 - tensorflow/contrib/factorization/BUILD | 31 +++++++++++++++++++ tensorflow/contrib/ffmpeg/BUILD | 2 ++ tensorflow/contrib/framework/BUILD | 17 ++++++++++ tensorflow/contrib/fused_conv/BUILD | 4 +-- tensorflow/contrib/gdr/BUILD | 1 - tensorflow/contrib/grid_rnn/BUILD | 4 +-- .../contrib/hvx/clock_cycle_profiling/BUILD | 4 --- tensorflow/contrib/image/BUILD | 4 ++- tensorflow/contrib/input_pipeline/BUILD | 2 +- tensorflow/contrib/layers/BUILD | 5 +++ tensorflow/contrib/linalg/BUILD | 11 ------- tensorflow/contrib/lookup/BUILD | 2 +- tensorflow/contrib/losses/BUILD | 25 ++++++++------- tensorflow/contrib/memory_stats/BUILD | 2 ++ tensorflow/contrib/meta_graph_transform/BUILD | 7 ++++- tensorflow/contrib/metrics/BUILD | 1 + tensorflow/contrib/nccl/BUILD | 5 +++ tensorflow/contrib/nearest_neighbor/BUILD | 22 +++++-------- tensorflow/contrib/nn/BUILD | 7 +++-- tensorflow/contrib/opt/BUILD | 6 ++-- tensorflow/contrib/predictor/BUILD | 6 ++-- tensorflow/contrib/quantize/BUILD | 4 ++- tensorflow/contrib/receptive_field/BUILD | 5 ++- tensorflow/contrib/reduce_slice_ops/BUILD | 1 + tensorflow/contrib/resampler/BUILD | 6 ++++ tensorflow/contrib/rnn/BUILD | 15 +++++---- tensorflow/contrib/saved_model/BUILD | 10 ++++-- tensorflow/contrib/seq2seq/BUILD | 13 ++++++++ tensorflow/contrib/session_bundle/BUILD | 3 -- tensorflow/contrib/signal/BUILD | 2 +- tensorflow/contrib/slim/BUILD | 2 -- .../contrib/slim/python/slim/data/BUILD | 3 +- tensorflow/contrib/stateless/BUILD | 2 +- tensorflow/contrib/summary/BUILD | 6 ++-- tensorflow/contrib/tensorboard/db/BUILD | 2 -- tensorflow/contrib/text/BUILD | 6 ++++ tensorflow/contrib/timeseries/examples/BUILD | 7 ++++- .../timeseries/python/timeseries/BUILD | 10 ++---- tensorflow/contrib/training/BUILD | 13 +++++--- 46 files changed, 227 insertions(+), 115 deletions(-) diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD index 35b9de27e7..8dff93b4f8 100644 --- a/tensorflow/contrib/all_reduce/BUILD +++ b/tensorflow/contrib/all_reduce/BUILD @@ -32,12 +32,17 @@ tf_py_test( additional_deps = [ ":all_reduce", "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:client", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:math_ops", "//tensorflow/python:constant_op", "//tensorflow/python:client_testlib", + "//tensorflow/python:platform", "//tensorflow/python:platform_test", + "//tensorflow/python:state_ops", ], ) diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD index ae3f48f1b2..8b7df4a84c 100644 --- a/tensorflow/contrib/batching/BUILD +++ b/tensorflow/contrib/batching/BUILD @@ -177,14 +177,13 @@ tf_custom_op_py_library( deps = [ ":batch_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", + "//tensorflow/python:gradients", "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", + "//tensorflow/python:script_ops", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/cloud/BUILD b/tensorflow/contrib/cloud/BUILD index eec2beddc4..aa8f5ed12b 100644 --- a/tensorflow/contrib/cloud/BUILD +++ b/tensorflow/contrib/cloud/BUILD @@ -63,11 +63,15 @@ tf_py_test( ":bigquery_reader_ops_op_lib", ":cloud_py", "//tensorflow/contrib/cloud/kernels:bigquery_reader_ops", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:data_flow_ops", + "//tensorflow/python:dtypes", "//tensorflow/python:io_ops", "//tensorflow/python:parsing_ops", + "//tensorflow/python:platform", + "//tensorflow/python:util", ], tags = ["manual"], ) diff --git a/tensorflow/contrib/cluster_resolver/BUILD b/tensorflow/contrib/cluster_resolver/BUILD index 9501c33245..15abd2be03 100644 --- a/tensorflow/contrib/cluster_resolver/BUILD +++ b/tensorflow/contrib/cluster_resolver/BUILD @@ -13,7 +13,9 @@ licenses(["notice"]) # Apache 2.0 filegroup( name = "all_files", srcs = glob( - ["**/*"], + include = [ + "**/*", + ], exclude = [ "**/METADATA", "**/OWNERS", @@ -37,9 +39,7 @@ py_library( py_library( name = "cluster_resolver_py", - srcs = [ - "python/training/cluster_resolver.py", - ], + srcs = ["python/training/cluster_resolver.py"], srcs_version = "PY2AND3", deps = [ "//tensorflow/python:training", @@ -48,9 +48,7 @@ py_library( py_library( name = "gce_cluster_resolver_py", - srcs = [ - "python/training/gce_cluster_resolver.py", - ], + srcs = ["python/training/gce_cluster_resolver.py"], srcs_version = "PY2AND3", deps = [ ":cluster_resolver_py", @@ -60,9 +58,7 @@ py_library( py_library( name = "tpu_cluster_resolver_py", - srcs = [ - "python/training/tpu_cluster_resolver.py", - ], + srcs = ["python/training/tpu_cluster_resolver.py"], srcs_version = "PY2AND3", deps = [ ":cluster_resolver_py", @@ -79,6 +75,7 @@ tf_py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:training", ], main = "python/training/cluster_resolver_test.py", ) @@ -88,11 +85,13 @@ tf_py_test( size = "small", srcs = ["python/training/gce_cluster_resolver_test.py"], additional_deps = [ + ":cluster_resolver_py", ":gce_cluster_resolver_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:training", ], main = "python/training/gce_cluster_resolver_test.py", ) @@ -107,6 +106,7 @@ tf_py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:training", ], main = "python/training/tpu_cluster_resolver_test.py", ) diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD index ae9413fdd6..f192f78b98 100644 --- a/tensorflow/contrib/cudnn_rnn/BUILD +++ b/tensorflow/contrib/cudnn_rnn/BUILD @@ -36,6 +36,7 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "//tensorflow/core:stream_executor", "//tensorflow/core/kernels:bounds_check_lib", "//third_party/eigen3", ], @@ -70,14 +71,23 @@ tf_custom_op_py_library( visibility = ["//visibility:public"], deps = [ ":cudnn_rnn_ops", + "//tensorflow/contrib/rnn:rnn_py", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", + "//tensorflow/python:common_shapes", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers_base", + "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:random_seed", + "//tensorflow/python:rnn_cell", "//tensorflow/python:state_ops", "//tensorflow/python:training", + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", ], ) @@ -104,9 +114,13 @@ tf_custom_op_py_library( "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers_base", "//tensorflow/python:platform", "//tensorflow/python:state_ops", "//tensorflow/python:training", + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", ], ) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index cb7b5cf462..96393f9f5a 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -67,6 +67,7 @@ py_test( deps = [ ":datasets", "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:script_ops", "//tensorflow/python/data", @@ -143,12 +144,11 @@ py_library( deps = [ "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:array_ops", - "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python/eager:context", @@ -169,7 +169,6 @@ py_test( "//tensorflow/python:lib", "//tensorflow/python:platform", "//tensorflow/python:training", - "//tensorflow/python:variables", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], @@ -190,6 +189,7 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python/eager:context", "//tensorflow/python/eager:function", + "@six_archive//:six", ], ) @@ -212,11 +212,9 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - "//tensorflow/python:framework_ops", "//tensorflow/python:layers_base", "//tensorflow/python:variable_scope", "//tensorflow/python/estimator:util", - "@six_archive//:six", ], ) @@ -227,9 +225,12 @@ py_test( deps = [ ":network", "//tensorflow/python:constant_op", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:layers", "//tensorflow/python:math_ops", "//tensorflow/python:nn_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:variable_scope", "//tensorflow/python/eager:test", ], ) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 79b166ac88..a0f83ac105 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -82,7 +82,6 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", - "//tensorflow/python:util", "//tensorflow/python/estimator", "//tensorflow/python/estimator:model_fn", "//tensorflow/python/estimator:util", diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD index 44095bd00a..fe86a20ab1 100644 --- a/tensorflow/contrib/factorization/BUILD +++ b/tensorflow/contrib/factorization/BUILD @@ -50,15 +50,22 @@ tf_custom_op_py_library( "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:linalg_ops", + "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:metrics", "//tensorflow/python:nn", "//tensorflow/python:platform", "//tensorflow/python:random_ops", "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", "//tensorflow/python:summary", + "//tensorflow/python:training", "//tensorflow/python:util", + "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/estimator", + "//tensorflow/python/estimator:model_fn", "//third_party/py/numpy", ], ) @@ -133,12 +140,17 @@ tf_py_test( ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", "//third_party/py/numpy", "//tensorflow/contrib/learn", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:data_flow_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform", "//tensorflow/python:platform_test", + "//tensorflow/python:random_ops", + "//tensorflow/python:random_seed", + "//tensorflow/python:training", ], tags = [ "no_pip", # b/38283730 @@ -162,6 +174,7 @@ tf_py_test( "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform", "//tensorflow/python:platform_test", + "//tensorflow/python:random_seed", "//tensorflow/python:variables", ], tags = ["notsan"], # b/62863147 @@ -193,10 +206,13 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", "//tensorflow/python:platform_test", + "//tensorflow/python:sparse_tensor", ], ) @@ -220,6 +236,7 @@ py_test( "//tensorflow/python:platform_benchmark", "//tensorflow/python:random_ops", "//tensorflow/python:training", + "//tensorflow/python/estimator:run_config", "//third_party/py/numpy", ], ) @@ -233,13 +250,20 @@ tf_py_test( ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", ":factorization_ops_test_utils_py", "//third_party/py/numpy", + "//tensorflow/contrib/learn", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:platform_benchmark", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", + "//tensorflow/python:training", "//tensorflow/python:variables", ], tags = [ @@ -256,11 +280,13 @@ tf_py_test( additional_deps = [ ":factorization_py", ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", + ":gen_factorization_ops", "//third_party/py/numpy", "//tensorflow/python:client_testlib", "//tensorflow/python:framework", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:sparse_tensor", ], ) @@ -284,10 +310,15 @@ tf_py_test( ":gen_factorization_ops", ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", "//third_party/py/numpy", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:sparse_tensor", ], ) diff --git a/tensorflow/contrib/ffmpeg/BUILD b/tensorflow/contrib/ffmpeg/BUILD index e205d92fbe..7a5a4cb8c9 100644 --- a/tensorflow/contrib/ffmpeg/BUILD +++ b/tensorflow/contrib/ffmpeg/BUILD @@ -89,6 +89,7 @@ tf_py_test( "@six_archive//:six", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", "//tensorflow/python:platform", ], data = [ @@ -105,6 +106,7 @@ tf_py_test( "@six_archive//:six", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", "//tensorflow/python:platform", ], data = [ diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD index 90aed3065b..891425fd8c 100644 --- a/tensorflow/contrib/framework/BUILD +++ b/tensorflow/contrib/framework/BUILD @@ -47,6 +47,7 @@ tf_custom_op_py_library( "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", "//tensorflow/python:audio_ops_gen", + "//tensorflow/python:check_ops", "//tensorflow/python:checkpoint_ops_gen", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework", @@ -56,13 +57,17 @@ tf_custom_op_py_library( "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", "//tensorflow/python:state_ops_gen", "//tensorflow/python:tensor_array_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", "//third_party/py/numpy", "@six_archive//:six", ], @@ -158,6 +163,11 @@ py_test( ":framework_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:gradients", + "//tensorflow/python:platform_test", + "//tensorflow/python:variables", + "//third_party/py/numpy", ], ) @@ -170,7 +180,14 @@ py_test( ":framework_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:gradients", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python/eager:backprop", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:tape", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/fused_conv/BUILD b/tensorflow/contrib/fused_conv/BUILD index 31917b40eb..ce37672895 100644 --- a/tensorflow/contrib/fused_conv/BUILD +++ b/tensorflow/contrib/fused_conv/BUILD @@ -38,7 +38,6 @@ tf_custom_op_py_library( ":fused_conv2d_bias_activation_op", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", @@ -49,6 +48,7 @@ tf_custom_op_py_library( "//tensorflow/python:nn_ops", "//tensorflow/python:platform", "//tensorflow/python:random_ops", + "//tensorflow/python:session", "//tensorflow/python:util", "//tensorflow/python:variables", "//third_party/py/numpy", @@ -69,7 +69,7 @@ tf_kernel_library( "//tensorflow/core:lib", "//tensorflow/core:lib_proto_parsing", "//tensorflow/core:stream_executor", - "//tensorflow/core/kernels:bounds_check_lib", + "//tensorflow/core/kernels:bounds_check", "//tensorflow/core/kernels:conv_2d_hdrs", "//tensorflow/core/kernels:conv_ops_gpu_hdrs", "//tensorflow/core/kernels:gpu_util_hdrs", diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD index a8053be69b..a417dba875 100644 --- a/tensorflow/contrib/gdr/BUILD +++ b/tensorflow/contrib/gdr/BUILD @@ -85,7 +85,6 @@ tf_cuda_library( "//tensorflow/core/distributed_runtime:rendezvous_mgr_interface", "//tensorflow/core/distributed_runtime:worker", "//tensorflow/core/distributed_runtime:worker_cache", - "//tensorflow/core/distributed_runtime:worker_env", "//tensorflow/core/distributed_runtime:worker_session", "//tensorflow/core/distributed_runtime/rpc:grpc_call", "//tensorflow/core/distributed_runtime/rpc:grpc_tensor_coding", diff --git a/tensorflow/contrib/grid_rnn/BUILD b/tensorflow/contrib/grid_rnn/BUILD index 7fbb9f024c..d601a1ec6f 100644 --- a/tensorflow/contrib/grid_rnn/BUILD +++ b/tensorflow/contrib/grid_rnn/BUILD @@ -31,14 +31,12 @@ cuda_py_tests( additional_deps = [ ":grid_rnn_py", "//third_party/py/numpy", - "//tensorflow/contrib/rnn:rnn_py", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", "//tensorflow/python:init_ops", "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", + "//tensorflow/python:rnn", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD index 8c92e33bdf..324035100d 100644 --- a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD +++ b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD @@ -52,13 +52,9 @@ tf_cc_binary( "//tensorflow/core:android_tensorflow_test_lib", ], "//conditions:default": [ - "//tensorflow/core:core_cpu", "//tensorflow/core:lib", - "//tensorflow/core:framework", "//tensorflow/core:framework_internal", - "//tensorflow/core:protos_all_cc", "//tensorflow/core:tensorflow", - "//tensorflow/core:test", ], }), ) diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD index c0c56d2e4a..157e97d237 100755 --- a/tensorflow/contrib/image/BUILD +++ b/tensorflow/contrib/image/BUILD @@ -75,11 +75,13 @@ tf_custom_op_py_library( ":image_ops", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", + "//tensorflow/python:common_shapes", "//tensorflow/python:constant_op", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/input_pipeline/BUILD b/tensorflow/contrib/input_pipeline/BUILD index bb7857eb99..9d6b4d5d87 100644 --- a/tensorflow/contrib/input_pipeline/BUILD +++ b/tensorflow/contrib/input_pipeline/BUILD @@ -67,9 +67,9 @@ tf_custom_op_py_library( "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:state_ops", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD index 1ae4d281c4..2f1f283811 100644 --- a/tensorflow/contrib/layers/BUILD +++ b/tensorflow/contrib/layers/BUILD @@ -88,17 +88,21 @@ tf_custom_op_py_library( "//tensorflow/python:clip_ops", "//tensorflow/python:common_shapes", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:data_flow_ops", "//tensorflow/python:embedding_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:init_ops", "//tensorflow/python:layers", + "//tensorflow/python:layers_base", + "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:random_ops", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:standard_ops", @@ -109,6 +113,7 @@ tf_custom_op_py_library( "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", "//tensorflow/python/feature_column", "@six_archive//:six", ], diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD index 734bac17dc..208e7bc69b 100644 --- a/tensorflow/contrib/linalg/BUILD +++ b/tensorflow/contrib/linalg/BUILD @@ -17,22 +17,11 @@ py_library( srcs = ["__init__.py"] + glob(["python/ops/*.py"]), srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:common_shapes", - "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//tensorflow/python/ops/linalg", - "//third_party/py/numpy", "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD index b8455477b0..b7b5418fe9 100644 --- a/tensorflow/contrib/lookup/BUILD +++ b/tensorflow/contrib/lookup/BUILD @@ -34,12 +34,12 @@ py_test( deps = [ ":lookup_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:lookup_ops", + "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", "//tensorflow/python:variables", diff --git a/tensorflow/contrib/losses/BUILD b/tensorflow/contrib/losses/BUILD index 33fbbe12d3..515290e217 100644 --- a/tensorflow/contrib/losses/BUILD +++ b/tensorflow/contrib/losses/BUILD @@ -19,12 +19,19 @@ py_library( ], srcs_version = "PY2AND3", deps = [ + ":metric_learning_py", "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", + "//tensorflow/python:script_ops", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:summary", "//tensorflow/python:util", ], ) @@ -59,13 +66,16 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", "//tensorflow/python:script_ops", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:summary", "//tensorflow/python:util", ], ) @@ -78,18 +88,11 @@ py_test( srcs_version = "PY2AND3", deps = [ ":metric_learning_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", + "//tensorflow/python:nn", + "//tensorflow/python:sparse_tensor", "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/memory_stats/BUILD b/tensorflow/contrib/memory_stats/BUILD index 8b9d30dcfd..72424c32e7 100644 --- a/tensorflow/contrib/memory_stats/BUILD +++ b/tensorflow/contrib/memory_stats/BUILD @@ -63,6 +63,8 @@ tf_custom_op_py_library( deps = [ ":memory_stats_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:platform", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/meta_graph_transform/BUILD b/tensorflow/contrib/meta_graph_transform/BUILD index d47ac5bcfe..4b5b1c3e15 100644 --- a/tensorflow/contrib/meta_graph_transform/BUILD +++ b/tensorflow/contrib/meta_graph_transform/BUILD @@ -21,7 +21,12 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/core:protos_all_py", - "//tensorflow/python:ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", + "//tensorflow/python:graph_util", + "//tensorflow/python:session", + "//tensorflow/python:training", + "//tensorflow/python:util", "//tensorflow/python/saved_model:constants", "//tensorflow/tools/graph_transforms:transform_graph_py", ], diff --git a/tensorflow/contrib/metrics/BUILD b/tensorflow/contrib/metrics/BUILD index e11dff08f8..9de664c822 100644 --- a/tensorflow/contrib/metrics/BUILD +++ b/tensorflow/contrib/metrics/BUILD @@ -42,6 +42,7 @@ py_library( "//tensorflow/python:state_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python:weights_broadcast_ops", ], ) diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD index 3aa3215a5f..ed9fb64b95 100644 --- a/tensorflow/contrib/nccl/BUILD +++ b/tensorflow/contrib/nccl/BUILD @@ -79,6 +79,7 @@ tf_kernel_library( "//tensorflow/core:gpu_headers_lib", "//tensorflow/core:lib", "//tensorflow/core:proto_text", + "//tensorflow/core:stream_executor", "@nccl_archive//:nccl", ], alwayslink = 1, @@ -114,7 +115,11 @@ tf_custom_op_py_library( deps = [ ":nccl_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:device", + "//tensorflow/python:framework_ops", "//tensorflow/python:platform", + "//tensorflow/python:util", + "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/contrib/nearest_neighbor/BUILD b/tensorflow/contrib/nearest_neighbor/BUILD index 84d59cc4be..9500c18b1d 100644 --- a/tensorflow/contrib/nearest_neighbor/BUILD +++ b/tensorflow/contrib/nearest_neighbor/BUILD @@ -41,18 +41,14 @@ tf_gen_op_wrapper_py( tf_custom_op_py_library( name = "nearest_neighbor_py", srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [ - ":python/ops/_nearest_neighbor_ops.so", - ], - kernels = [ - ":nearest_neighbor_ops_kernels", - ], + dso = [":python/ops/_nearest_neighbor_ops.so"], + kernels = [":nearest_neighbor_ops_kernels"], srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - ":nearest_neighbor_ops_pywrapper", "//tensorflow/contrib/util:util_py", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", ], ) @@ -70,9 +66,7 @@ tf_kernel_library( cc_library( name = "heap", - hdrs = [ - "kernels/heap.h", - ], + hdrs = ["kernels/heap.h"], ) tf_cc_test( @@ -81,17 +75,14 @@ tf_cc_test( srcs = ["kernels/heap_test.cc"], deps = [ ":heap", - "//tensorflow/core:test", "//tensorflow/core:test_main", - "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", ], ) cc_library( name = "hyperplane_lsh_probes", - hdrs = [ - "kernels/hyperplane_lsh_probes.h", - ], + hdrs = ["kernels/hyperplane_lsh_probes.h"], deps = [ ":heap", "//third_party/eigen3", @@ -107,6 +98,7 @@ tf_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", ], ) diff --git a/tensorflow/contrib/nn/BUILD b/tensorflow/contrib/nn/BUILD index 0ed7e52159..56a24ac77f 100644 --- a/tensorflow/contrib/nn/BUILD +++ b/tensorflow/contrib/nn/BUILD @@ -30,6 +30,7 @@ py_library( "//tensorflow/python:function", "//tensorflow/python:math_ops", "//tensorflow/python:nn", + "//tensorflow/python:nn_ops", "//tensorflow/python:random_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", @@ -77,9 +78,9 @@ py_test( deps = [ ":nn_py", "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn", + "//tensorflow/python:constant_op", + "//tensorflow/python:gradient_checker", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index 8b2b31d5bc..096d2270e4 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -86,9 +86,9 @@ py_test( ], deps = [ ":opt_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variables", "//third_party/py/numpy", @@ -119,13 +119,13 @@ py_test( deps = [ ":opt_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:session", "//tensorflow/python:variables", "//third_party/py/numpy", ], @@ -139,9 +139,11 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:client", "//tensorflow/python:client_testlib", + "//tensorflow/python:data_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/predictor/BUILD b/tensorflow/contrib/predictor/BUILD index 745dc2f836..1bf40ab6b2 100644 --- a/tensorflow/contrib/predictor/BUILD +++ b/tensorflow/contrib/predictor/BUILD @@ -25,7 +25,10 @@ py_library( srcs = ["__init__.py"], srcs_version = "PY2AND3", visibility = ["//visibility:public"], - deps = [":predictor_factories"], + deps = [ + ":predictor_factories", + "//tensorflow/python:util", + ], ) py_library( @@ -58,7 +61,6 @@ py_library( "//tensorflow/python:session", "//tensorflow/python/saved_model:loader", "//tensorflow/python/saved_model:signature_constants", - "//tensorflow/python/saved_model:signature_def_utils", ], ) diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD index 2c0ffaf6c0..935af80e7a 100644 --- a/tensorflow/contrib/quantize/BUILD +++ b/tensorflow/contrib/quantize/BUILD @@ -29,6 +29,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":graph_matcher", + "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", @@ -75,6 +76,7 @@ py_library( ":input_to_ops", "//tensorflow/contrib/graph_editor:graph_editor_py", "//tensorflow/python:array_ops", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", @@ -87,7 +89,6 @@ py_test( srcs_version = "PY2AND3", deps = [ ":fold_batch_norms", - ":graph_matcher", "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", @@ -208,6 +209,7 @@ py_library( ":fold_batch_norms", ":quantize", "//tensorflow/python:framework_ops", + "//tensorflow/python:util", "//tensorflow/python:variables", ], ) diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD index ed2f3af08c..d16b2908a0 100644 --- a/tensorflow/contrib/receptive_field/BUILD +++ b/tensorflow/contrib/receptive_field/BUILD @@ -39,7 +39,9 @@ py_library( deps = [ ":graph_compute_order_py", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:framework_ops", "//tensorflow/python:platform", + "//third_party/py/numpy", ], ) @@ -49,12 +51,13 @@ py_test( srcs_version = "PY2AND3", deps = [ ":receptive_field_py", - "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/slim", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", "//tensorflow/python:nn", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/reduce_slice_ops/BUILD b/tensorflow/contrib/reduce_slice_ops/BUILD index fded03090e..b31f4488f5 100644 --- a/tensorflow/contrib/reduce_slice_ops/BUILD +++ b/tensorflow/contrib/reduce_slice_ops/BUILD @@ -71,6 +71,7 @@ tf_custom_op_py_library( ":reduce_slice_ops", "//tensorflow/contrib/util:util_py", "//tensorflow/python:framework", + "//tensorflow/python:platform", ], ) diff --git a/tensorflow/contrib/resampler/BUILD b/tensorflow/contrib/resampler/BUILD index 1b9efd1ecd..f0ecc8b85a 100644 --- a/tensorflow/contrib/resampler/BUILD +++ b/tensorflow/contrib/resampler/BUILD @@ -26,9 +26,15 @@ tf_custom_op_py_library( deps = [ ":resampler_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", "//tensorflow/python:framework_ops", "//tensorflow/python:platform", "//tensorflow/python:util", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD index 29ba26d75d..b70a5bbcd1 100644 --- a/tensorflow/contrib/rnn/BUILD +++ b/tensorflow/contrib/rnn/BUILD @@ -62,21 +62,24 @@ tf_custom_op_py_library( "//tensorflow/contrib/compiler:compiler_py", "//tensorflow/contrib/layers:layers_py", "//tensorflow/contrib/util:util_py", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:clip_ops", "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:init_ops", - "//tensorflow/python:layers", "//tensorflow/python:math_ops", "//tensorflow/python:nn_ops", - "//tensorflow/python:partitioned_variables", "//tensorflow/python:platform", + "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:random_ops", "//tensorflow/python:rnn", "//tensorflow/python:rnn_cell", + "//tensorflow/python:session", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", ], ) @@ -382,7 +385,6 @@ py_binary( srcs_version = "PY2AND3", deps = [ "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", "//tensorflow/python:framework_ops", "//tensorflow/python:platform", "//tensorflow/python:pywrap_tensorflow", @@ -412,8 +414,5 @@ py_library( name = "benchmarking", srcs = ["python/kernel_tests/benchmarking.py"], srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_ops", - "//third_party/py/numpy", - ], + deps = ["//tensorflow/python:framework_ops"], ) diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD index a82ee6ac41..20be819e07 100644 --- a/tensorflow/contrib/saved_model/BUILD +++ b/tensorflow/contrib/saved_model/BUILD @@ -37,9 +37,14 @@ py_library( visibility = ["//visibility:public"], deps = [ "//tensorflow/core:protos_all_py", + "//tensorflow/python:framework_ops", "//tensorflow/python:lib", "//tensorflow/python:util", + "//tensorflow/python/saved_model:builder", "//tensorflow/python/saved_model:constants", + "//tensorflow/python/saved_model:signature_constants", + "//tensorflow/python/saved_model:signature_def_utils", + "//tensorflow/python/saved_model:tag_constants", ], ) @@ -85,10 +90,11 @@ py_test( deps = [ ":saved_model_py", "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", - "//tensorflow/python/saved_model", + "//tensorflow/python:variables", + "//tensorflow/python/saved_model:loader", "//tensorflow/python/saved_model:signature_constants", + "//tensorflow/python/saved_model:tag_constants", ], ) diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD index f1e39a1373..ab80c68b1a 100644 --- a/tensorflow/contrib/seq2seq/BUILD +++ b/tensorflow/contrib/seq2seq/BUILD @@ -33,18 +33,31 @@ tf_custom_op_py_library( "//tensorflow/contrib/distributions:distributions_py", "//tensorflow/contrib/layers:layers_py", "//tensorflow/contrib/rnn:rnn_py", + "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", + "//tensorflow/python:check_ops", + "//tensorflow/python:clip_ops", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:functional_ops", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers", + "//tensorflow/python:layers_base", "//tensorflow/python:math_ops", + "//tensorflow/python:nn_ops", "//tensorflow/python:platform", + "//tensorflow/python:random_ops", "//tensorflow/python:rnn", "//tensorflow/python:rnn_cell", "//tensorflow/python:script_ops", "//tensorflow/python:tensor_array_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python/ops/distributions", "//third_party/py/numpy", + "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD index 8a1c9ba0a2..67011c8fef 100644 --- a/tensorflow/contrib/session_bundle/BUILD +++ b/tensorflow/contrib/session_bundle/BUILD @@ -136,7 +136,6 @@ py_test( ":gc", ":manifest_proto_py", "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", @@ -411,8 +410,6 @@ tf_cc_test( ":test_util", "//tensorflow/cc/saved_model:signature_constants", "//tensorflow/cc/saved_model:tag_constants", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", diff --git a/tensorflow/contrib/signal/BUILD b/tensorflow/contrib/signal/BUILD index 2204b684ac..b67090dd50 100644 --- a/tensorflow/contrib/signal/BUILD +++ b/tensorflow/contrib/signal/BUILD @@ -32,8 +32,8 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework", "//tensorflow/python:tf_optimizer", + "//tensorflow/python:training", ], ) diff --git a/tensorflow/contrib/slim/BUILD b/tensorflow/contrib/slim/BUILD index d2664b612c..23c23af2f4 100644 --- a/tensorflow/contrib/slim/BUILD +++ b/tensorflow/contrib/slim/BUILD @@ -48,7 +48,6 @@ py_library( srcs = ["python/slim/learning.py"], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/training:training_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:client", @@ -78,7 +77,6 @@ py_test( "//tensorflow/contrib/losses:losses_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD index fc71a5fe41..5daabbd62e 100644 --- a/tensorflow/contrib/slim/python/slim/data/BUILD +++ b/tensorflow/contrib/slim/python/slim/data/BUILD @@ -68,13 +68,13 @@ py_test( ":tfexample_decoder", "//tensorflow/contrib/slim:queues", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:image_ops", "//tensorflow/python:io_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", + "//tensorflow/python:session", ], ) @@ -187,6 +187,7 @@ py_test( "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:image_ops", + "//tensorflow/python:lookup_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", "//third_party/py/numpy", diff --git a/tensorflow/contrib/stateless/BUILD b/tensorflow/contrib/stateless/BUILD index 865fb72a55..6e259e1d32 100644 --- a/tensorflow/contrib/stateless/BUILD +++ b/tensorflow/contrib/stateless/BUILD @@ -21,7 +21,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":stateless_random_ops", - "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD index 8cb5c3f381..da23f1c380 100644 --- a/tensorflow/contrib/summary/BUILD +++ b/tensorflow/contrib/summary/BUILD @@ -26,9 +26,8 @@ py_test( deps = [ ":summary_ops", ":summary_test_util", - "//tensorflow/core:protos_all_py", + "//tensorflow/python:errors", "//tensorflow/python:framework_test_lib", - "//tensorflow/python:lib", "//tensorflow/python:platform", "//tensorflow/python:training", "//tensorflow/python/eager:function", @@ -43,12 +42,15 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ ":gen_summary_ops", + "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:layers_base", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:summary_op_util", "//tensorflow/python:training", + "//tensorflow/python:util", "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/contrib/tensorboard/db/BUILD index f056632295..fb2d54916b 100644 --- a/tensorflow/contrib/tensorboard/db/BUILD +++ b/tensorflow/contrib/tensorboard/db/BUILD @@ -22,10 +22,8 @@ tf_cc_test( srcs = ["schema_test.cc"], deps = [ ":schema", - "//tensorflow/core:lib", "//tensorflow/core:test", "//tensorflow/core:test_main", - "//tensorflow/core/lib/db:sqlite", ], ) diff --git a/tensorflow/contrib/text/BUILD b/tensorflow/contrib/text/BUILD index 8a2cb28684..698fdd830f 100644 --- a/tensorflow/contrib/text/BUILD +++ b/tensorflow/contrib/text/BUILD @@ -36,15 +36,21 @@ tf_custom_op_py_library( srcs_version = "PY2AND3", deps = [ ":gen_skip_gram_ops", + "//tensorflow/contrib/lookup:lookup_py", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:dtypes", "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:ops", "//tensorflow/python:platform", + "//tensorflow/python:random_ops", + "//tensorflow/python:random_seed", "//tensorflow/python:training", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD index 222a77c489..755b0657e9 100644 --- a/tensorflow/contrib/timeseries/examples/BUILD +++ b/tensorflow/contrib/timeseries/examples/BUILD @@ -88,6 +88,8 @@ py_binary( tags = ["no_pip"], deps = [ "//tensorflow:tensorflow_py", + "//tensorflow/contrib/timeseries/python/timeseries:estimators", + "//tensorflow/contrib/timeseries/python/timeseries:model", "//third_party/py/numpy", ], ) @@ -98,7 +100,10 @@ py_test( srcs = ["lstm_test.py"], srcs_version = "PY2AND3", tags = ["notsan"], - deps = [":lstm"], + deps = [ + ":lstm", + "//tensorflow/python:client_testlib", + ], ) filegroup( diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index 7491b1b2d2..5f04eb2f5a 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -138,15 +138,13 @@ py_library( "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", "//tensorflow/python:state_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python/estimator:estimator_py", "//tensorflow/python/estimator:export", - "//third_party/py/numpy", + "//tensorflow/python/estimator:head", ], ) @@ -184,7 +182,6 @@ py_library( srcs_version = "PY2AND3", deps = [ ":feature_keys", - "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:init_ops", @@ -207,7 +204,6 @@ py_test( ":model_utils", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:variables", ], ) @@ -327,11 +323,11 @@ py_library( ":input_pipeline", ":state_management", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:random_seed", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variables", @@ -380,10 +376,10 @@ py_test( ":input_pipeline", ":test_utils", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_ops", "//tensorflow/python:platform", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD index 0df5ff50c0..6139c1d583 100644 --- a/tensorflow/contrib/training/BUILD +++ b/tensorflow/contrib/training/BUILD @@ -42,6 +42,7 @@ py_library( "//tensorflow/python:data_flow_ops", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:layers_base", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", @@ -112,6 +113,7 @@ py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", + "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", "//tensorflow/python:training", @@ -126,9 +128,12 @@ py_test( srcs = ["python/training/feeding_queue_runner_test.py"], srcs_version = "PY2AND3", deps = [ - ":training_py", "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:session", + "//tensorflow/python:training", + "//tensorflow/python/estimator:inputs_queues", + "//third_party/py/numpy", ], ) @@ -140,7 +145,6 @@ py_test( deps = [ ":training_py", "//tensorflow/python:client_testlib", - "@six_archive//:six", ], ) @@ -244,12 +248,12 @@ py_test( "//tensorflow/contrib/metrics:metrics_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:random_seed", + "//tensorflow/python:session", "//tensorflow/python:state_ops", "//tensorflow/python:summary", "//tensorflow/python:training", @@ -271,6 +275,7 @@ py_test( "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:gradients", "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:random_seed", -- GitLab From ce0238198052358d102ca7786ad9be60a5e76d28 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 30 Oct 2017 08:07:11 -0700 Subject: [PATCH 506/573] Add ability to fetch return nodes and unused input mappings from C API GraphDef import This change introduces yet another ImportGraphDef function to the C API (TF_GraphImportGraphDefWithResults), but this one has extensible return values so we shouldn't have to add more in the future. This change also modifies the ImportGraphDef C interface to manage all string data for the user. PiperOrigin-RevId: 173894710 --- tensorflow/c/c_api.cc | 227 +++++++++++++++------ tensorflow/c/c_api.h | 57 +++++- tensorflow/c/c_api_internal.h | 16 ++ tensorflow/c/c_api_test.cc | 135 +++++++++++- tensorflow/core/graph/graph_constructor.cc | 2 +- tensorflow/core/graph/graph_constructor.h | 2 +- 6 files changed, 362 insertions(+), 77 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index cd98393e0a..b43d202f4e 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -86,6 +86,7 @@ using tensorflow::errors::FailedPrecondition; using tensorflow::errors::InvalidArgument; using tensorflow::gtl::ArraySlice; using tensorflow::mutex_lock; +using tensorflow::string; using tensorflow::strings::StrCat; extern "C" { @@ -366,7 +367,7 @@ namespace { // Reset helper for converting character arrays to string vectors. void TF_Reset_Helper(const TF_SessionOptions* opt, const char** containers, int ncontainers, TF_Status* status) { - std::vector container_names(ncontainers); + std::vector container_names(ncontainers); for (int i = 0; i < ncontainers; ++i) { container_names[i] = containers[i]; } @@ -482,7 +483,7 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst) { const char* limit = input + src_size; *dst = Tensor(static_cast(src->dtype), src->shape); - auto dstarray = dst->flat(); + auto dstarray = dst->flat(); for (tensorflow::int64 i = 0; i < num_elements; ++i) { tensorflow::uint64 offset = reinterpret_cast(input)[i]; @@ -556,9 +557,9 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, // Compute bytes needed for encoding. size_t size = 0; - const auto& srcarray = src.flat(); + const auto& srcarray = src.flat(); for (int i = 0; i < srcarray.size(); ++i) { - const tensorflow::string& s = srcarray(i); + const string& s = srcarray(i); // uint64 starting_offset, TF_StringEncode-d string. size += sizeof(tensorflow::uint64) + TF_StringEncodedSize(s.size()); } @@ -572,7 +573,7 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, for (int i = 0; i < srcarray.size(); ++i) { *offsets = (dst - data_start); offsets++; - const tensorflow::string& s = srcarray(i); + const string& s = srcarray(i); size_t consumed = TF_StringEncode(s.data(), s.size(), dst, dst_len, status); if (!status->status.ok()) { status->status = InvalidArgument( @@ -637,10 +638,9 @@ static void TF_Run_Setup(int noutputs, TF_Tensor** c_outputs, } } -static bool TF_Run_Inputs( - TF_Tensor* const* c_inputs, - std::vector>* input_pairs, - TF_Status* status) { +static bool TF_Run_Inputs(TF_Tensor* const* c_inputs, + std::vector>* input_pairs, + TF_Status* status) { const int ninputs = input_pairs->size(); for (int i = 0; i < ninputs; ++i) { status->status = TF_TensorToTensor(c_inputs[i], &(*input_pairs)[i].second); @@ -652,13 +652,12 @@ static bool TF_Run_Inputs( static void TF_Run_Helper( Session* session, const char* handle, const TF_Buffer* run_options, // Input tensors - const std::vector>& input_pairs, + const std::vector>& input_pairs, // Output tensors - const std::vector& output_tensor_names, - TF_Tensor** c_outputs, + const std::vector& output_tensor_names, TF_Tensor** c_outputs, // Target nodes - const std::vector& target_oper_names, - TF_Buffer* run_metadata, TF_Status* status) { + const std::vector& target_oper_names, TF_Buffer* run_metadata, + TF_Status* status) { const int noutputs = output_tensor_names.size(); std::vector outputs(noutputs); Status result; @@ -718,16 +717,16 @@ void TF_Run(TF_DeprecatedSession* s, const TF_Buffer* run_options, const char** c_target_oper_names, int ntargets, TF_Buffer* run_metadata, TF_Status* status) { TF_Run_Setup(noutputs, c_outputs, status); - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(c_inputs, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = c_input_names[i]; } - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = c_output_names[i]; } - std::vector target_oper_names(ntargets); + std::vector target_oper_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_oper_names[i] = c_target_oper_names[i]; } @@ -745,9 +744,9 @@ void TF_PRunSetup(TF_DeprecatedSession* s, const char** handle, TF_Status* status) { *handle = nullptr; - std::vector input_names(ninputs); - std::vector output_names(noutputs); - std::vector target_oper_names(ntargets); + std::vector input_names(ninputs); + std::vector output_names(noutputs); + std::vector target_oper_names(ntargets); for (int i = 0; i < ninputs; ++i) { input_names[i] = c_input_names[i]; } @@ -757,7 +756,7 @@ void TF_PRunSetup(TF_DeprecatedSession* s, for (int i = 0; i < ntargets; ++i) { target_oper_names[i] = c_target_oper_names[i]; } - tensorflow::string new_handle; + string new_handle; status->status = s->session->PRunSetup(input_names, output_names, target_oper_names, &new_handle); if (status->status.ok()) { @@ -776,17 +775,17 @@ void TF_PRun(TF_DeprecatedSession* s, const char* handle, const char** c_target_oper_names, int ntargets, TF_Status* status) { TF_Run_Setup(noutputs, c_outputs, status); - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(c_inputs, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = c_input_names[i]; } - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = c_output_names[i]; } - std::vector target_oper_names(ntargets); + std::vector target_oper_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_oper_names[i] = c_target_oper_names[i]; } @@ -881,7 +880,7 @@ TF_Operation* ToOperation(Node* node) { return static_cast(static_cast(node)); } -tensorflow::string OutputName(const TF_Output& output) { +string OutputName(const TF_Output& output) { return StrCat(output.oper->node.name(), ":", output.index); } @@ -1254,7 +1253,7 @@ void TF_SetAttrValueProto(TF_OperationDescription* desc, const char* attr_name, return; } desc->colocation_constraints.clear(); - for (const tensorflow::string& location : attr_value.list().s()) { + for (const string& location : attr_value.list().s()) { desc->colocation_constraints.insert(location); } } else { @@ -1276,8 +1275,8 @@ static TF_Operation* TF_FinishOperationLocked(TF_OperationDescription* desc, if (!desc->colocation_constraints.empty()) { desc->node_builder.Attr( tensorflow::kColocationAttrName, - std::vector(desc->colocation_constraints.begin(), - desc->colocation_constraints.end())); + std::vector(desc->colocation_constraints.begin(), + desc->colocation_constraints.end())); } status->status = desc->node_builder.Finalize(&desc->graph->graph, &ret); @@ -1500,7 +1499,7 @@ TF_AttrMetadata TF_OperationGetAttrMetadata(TF_Operation* oper, for (int i = 0; i < oper->node.op_def().attr_size(); ++i) { const auto& a = oper->node.op_def().attr(i); if (a.name().compare(attr_name) != 0) continue; - const tensorflow::string& typestr = a.type(); + const string& typestr = a.type(); if (typestr == "list(string)") { metadata.type = TF_ATTR_STRING; } else if (typestr == "list(int)") { @@ -1580,7 +1579,7 @@ void TF_OperationGetAttrStringList(TF_Operation* oper, const char* attr_name, const auto len = std::min(max_values, attr->list().s_size()); char* p = static_cast(storage); for (int i = 0; i < len; ++i) { - const tensorflow::string& s = attr->list().s(i); + const string& s = attr->list().s(i); values[i] = p; lengths[i] = s.size(); if ((p + s.size()) > (static_cast(storage) + storage_size)) { @@ -1824,7 +1823,11 @@ void TF_ImportGraphDefOptionsSetPrefix(TF_ImportGraphDefOptions* opts, void TF_ImportGraphDefOptionsAddInputMapping(TF_ImportGraphDefOptions* opts, const char* src_name, int src_index, TF_Output dst) { - opts->opts.input_map[TensorId(src_name, src_index)] = ToTensorId(dst); + opts->tensor_id_data.push_back(src_name); + const string& src_name_str = opts->tensor_id_data.back(); + // We don't need to store dst's name in tensor_id_data, since `dst` must + // outlive the ImportGraphDef call. + opts->opts.input_map[TensorId(src_name_str, src_index)] = ToTensorId(dst); } void TF_ImportGraphDefOptionsRemapControlDependency( @@ -1840,7 +1843,9 @@ extern void TF_ImportGraphDefOptionsAddControlDependency( void TF_ImportGraphDefOptionsAddReturnOutput(TF_ImportGraphDefOptions* opts, const char* oper_name, int index) { - opts->opts.return_tensors.push_back({oper_name, index}); + opts->tensor_id_data.push_back(oper_name); + const string& oper_name_str = opts->tensor_id_data.back(); + opts->opts.return_tensors.emplace_back(oper_name_str, index); } int TF_ImportGraphDefOptionsNumReturnOutputs( @@ -1848,57 +1853,142 @@ int TF_ImportGraphDefOptionsNumReturnOutputs( return opts->opts.return_tensors.size(); } +void TF_ImportGraphDefOptionsAddReturnOperation(TF_ImportGraphDefOptions* opts, + const char* oper_name) { + opts->opts.return_nodes.push_back(oper_name); +} + +int TF_ImportGraphDefOptionsNumReturnOperations( + const TF_ImportGraphDefOptions* opts) { + return opts->opts.return_nodes.size(); +} + +void TF_ImportGraphDefResultsReturnOutputs(TF_ImportGraphDefResults* results, + int* num_outputs, + TF_Output** outputs) { + *num_outputs = results->return_tensors.size(); + *outputs = results->return_tensors.data(); +} + +void TF_ImportGraphDefResultsReturnOperations(TF_ImportGraphDefResults* results, + int* num_opers, + TF_Operation*** opers) { + *num_opers = results->return_nodes.size(); + *opers = results->return_nodes.data(); +} + +void TF_ImportGraphDefResultsUnusedInputMappings( + TF_ImportGraphDefResults* results, int* num_unused_input_mappings, + const char*** src_names, int** src_indexes) { + *num_unused_input_mappings = results->unused_key_names.size(); + *src_names = results->unused_key_names.data(); + *src_indexes = results->unused_key_indexes.data(); +} + +void TF_DeleteImportGraphDefResults(TF_ImportGraphDefResults* results) { + delete results; +} + static void GraphImportGraphDefLocked(TF_Graph* graph, const GraphDef& def, const TF_ImportGraphDefOptions* opts, - TF_Output* return_outputs, - int num_return_outputs, TF_Status* status) + TF_ImportGraphDefResults* tf_results, + TF_Status* status) EXCLUSIVE_LOCKS_REQUIRED(graph->mu) { - if (num_return_outputs != opts->opts.return_tensors.size()) { - status->status = InvalidArgument("Expected 'num_return_outputs' to be ", - opts->opts.return_tensors.size(), ", got ", - num_return_outputs); - return; - } - if (num_return_outputs > 0 && return_outputs == nullptr) { - status->status = InvalidArgument( - "'return_outputs' must be preallocated to length ", num_return_outputs); - return; - } const int last_node_id = graph->graph.num_node_ids(); tensorflow::ImportGraphDefResults results; status->status = tensorflow::ImportGraphDef(opts->opts, def, &graph->graph, &graph->refiner, &results); if (!status->status.ok()) return; + + // Add new nodes to name_map for (int i = last_node_id; i < graph->graph.num_node_ids(); ++i) { auto* node = graph->graph.FindNodeId(i); if (node != nullptr) graph->name_map[node->name()] = node; } - DCHECK_EQ(results.return_tensors.size(), num_return_outputs); - for (int i = 0; i < num_return_outputs; ++i) { - return_outputs[i].oper = ToOperation(results.return_tensors[i].first); - return_outputs[i].index = results.return_tensors[i].second; + + // Populate return_tensors + DCHECK(tf_results->return_tensors.empty()); + tf_results->return_tensors.resize(results.return_tensors.size()); + for (int i = 0; i < results.return_tensors.size(); ++i) { + tf_results->return_tensors[i].oper = + ToOperation(results.return_tensors[i].first); + tf_results->return_tensors[i].index = results.return_tensors[i].second; + } + + // Populate return_nodes + DCHECK(tf_results->return_nodes.empty()); + tf_results->return_nodes.resize(results.return_nodes.size()); + for (int i = 0; i < results.return_nodes.size(); ++i) { + tf_results->return_nodes[i] = ToOperation(results.return_nodes[i]); + } + + // Populate unused map keys + DCHECK(tf_results->unused_key_names.empty()); + DCHECK(tf_results->unused_key_indexes.empty()); + DCHECK(tf_results->unused_key_names_data.empty()); + tf_results->unused_key_names.resize(results.unused_input_map_keys.size()); + tf_results->unused_key_indexes.resize(results.unused_input_map_keys.size()); + for (int i = 0; i < results.unused_input_map_keys.size(); ++i) { + TensorId id = results.unused_input_map_keys[i]; + tf_results->unused_key_names_data.push_back(id.first.ToString()); + tf_results->unused_key_names[i] = + tf_results->unused_key_names_data.back().c_str(); + tf_results->unused_key_indexes[i] = id.second; + } +} + +TF_ImportGraphDefResults* TF_GraphImportGraphDefWithResults( + TF_Graph* graph, const TF_Buffer* graph_def, + const TF_ImportGraphDefOptions* options, TF_Status* status) { + GraphDef def; + if (!def.ParseFromArray(graph_def->data, graph_def->length)) { + status->status = InvalidArgument("Invalid GraphDef"); + return nullptr; } + auto results = new TF_ImportGraphDefResults(); + mutex_lock l(graph->mu); + GraphImportGraphDefLocked(graph, def, options, results, status); + if (!status->status.ok()) { + delete results; + return nullptr; + } + return results; } void TF_GraphImportGraphDefWithReturnOutputs( TF_Graph* graph, const TF_Buffer* graph_def, - const TF_ImportGraphDefOptions* opts, TF_Output* return_outputs, + const TF_ImportGraphDefOptions* options, TF_Output* return_outputs, int num_return_outputs, TF_Status* status) { + if (num_return_outputs != options->opts.return_tensors.size()) { + status->status = InvalidArgument("Expected 'num_return_outputs' to be ", + options->opts.return_tensors.size(), + ", got ", num_return_outputs); + return; + } + if (num_return_outputs > 0 && return_outputs == nullptr) { + status->status = InvalidArgument( + "'return_outputs' must be preallocated to length ", num_return_outputs); + return; + } GraphDef def; if (!def.ParseFromArray(graph_def->data, graph_def->length)) { status->status = InvalidArgument("Invalid GraphDef"); return; } + TF_ImportGraphDefResults results; mutex_lock l(graph->mu); - GraphImportGraphDefLocked(graph, def, opts, return_outputs, - num_return_outputs, status); + GraphImportGraphDefLocked(graph, def, options, &results, status); + DCHECK_EQ(results.return_tensors.size(), num_return_outputs); + memcpy(return_outputs, results.return_tensors.data(), + num_return_outputs * sizeof(TF_Output)); } void TF_GraphImportGraphDef(TF_Graph* graph, const TF_Buffer* graph_def, const TF_ImportGraphDefOptions* options, TF_Status* status) { - TF_GraphImportGraphDefWithReturnOutputs(graph, graph_def, options, nullptr, 0, - status); + TF_ImportGraphDefResults* results = + TF_GraphImportGraphDefWithResults(graph, graph_def, options, status); + TF_DeleteImportGraphDefResults(results); } // While loop functions ------------------------------------------------------- @@ -1930,7 +2020,7 @@ Status CopyGraph(Graph* src_graph, Graph* dst_graph, tensorflow::ShapeRefiner* dst_refiner, const TF_Output* src_inputs, const std::vector& dst_inputs, - const tensorflow::string& prefix, + const string& prefix, const std::vector& control_deps, const TF_Output* nodes_to_return, int nreturn_nodes, std::vector* return_nodes) { @@ -2257,9 +2347,9 @@ TF_Session* TF_LoadSessionFromSavedModel( return nullptr; } - std::unordered_set tag_set; + std::unordered_set tag_set; for (int i = 0; i < tags_len; i++) { - tag_set.insert(tensorflow::string(tags[i])); + tag_set.insert(string(tags[i])); } tensorflow::SavedModelBundle bundle; @@ -2275,8 +2365,9 @@ TF_Session* TF_LoadSessionFromSavedModel( // TODO(jhseu): When Session is modified to take Graphs instead of // GraphDefs, return the Graph generated in LoadSavedModel(). TF_ImportGraphDefOptions* import_opts = TF_NewImportGraphDefOptions(); + TF_ImportGraphDefResults results; GraphImportGraphDefLocked(graph, bundle.meta_graph_def.graph_def(), - import_opts, nullptr, 0, status); + import_opts, &results, status); TF_DeleteImportGraphDefOptions(import_opts); if (TF_GetCode(status) != TF_OK) return nullptr; @@ -2372,20 +2463,20 @@ void TF_SessionRun(TF_Session* session, const TF_Buffer* run_options, TF_Run_Setup(noutputs, output_values, status); // Convert from TF_Output and TF_Tensor to a string and Tensor. - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(input_values, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = OutputName(inputs[i]); } // Convert from TF_Output to string names. - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = OutputName(outputs[i]); } // Convert from TF_Operation* to string names. - std::vector target_names(ntargets); + std::vector target_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_names[i] = target_opers[i]->node.name(); } @@ -2406,22 +2497,22 @@ void TF_SessionPRunSetup(TF_Session* session, const TF_Output* inputs, return; } - std::vector input_names(ninputs); + std::vector input_names(ninputs); for (int i = 0; i < ninputs; ++i) { input_names[i] = OutputName(inputs[i]); } - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = OutputName(outputs[i]); } - std::vector target_names(ntargets); + std::vector target_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_names[i] = target_opers[i]->node.name(); } - tensorflow::string new_handle; + string new_handle; status->status = session->session->PRunSetup(input_names, output_names, target_names, &new_handle); if (status->status.ok()) { @@ -2452,20 +2543,20 @@ void TF_SessionPRun(TF_Session* session, const char* handle, TF_Run_Setup(noutputs, output_values, status); // Convert from TF_Output and TF_Tensor to a string and Tensor. - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(input_values, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = OutputName(inputs[i]); } // Convert from TF_Output to string names. - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = OutputName(outputs[i]); } // Convert from TF_Operation* to string names. - std::vector target_names(ntargets); + std::vector target_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_names[i] = target_opers[i]->node.name(); } diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index 1e8bfdc7b0..ca5c934634 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -914,7 +914,62 @@ TF_CAPI_EXPORT extern void TF_ImportGraphDefOptionsAddReturnOutput( TF_CAPI_EXPORT extern int TF_ImportGraphDefOptionsNumReturnOutputs( const TF_ImportGraphDefOptions* opts); +// Add an operation in `graph_def` to be returned via the `return_opers` output +// parameter of TF_GraphImportGraphDef(). +TF_CAPI_EXPORT extern void TF_ImportGraphDefOptionsAddReturnOperation( + TF_ImportGraphDefOptions* opts, const char* oper_name); + +// Returns the number of return operations added via +// TF_ImportGraphDefOptionsAddReturnOperation(). +TF_CAPI_EXPORT extern int TF_ImportGraphDefOptionsNumReturnOperations( + const TF_ImportGraphDefOptions* opts); + +// TF_ImportGraphDefResults holds results that are generated by +// TF_GraphImportGraphDefWithResults(). +typedef struct TF_ImportGraphDefResults TF_ImportGraphDefResults; + +// Fetches the return outputs requested via +// TF_ImportGraphDefOptionsAddReturnOutput(). The number of fetched outputs is +// returned in `num_outputs`. The array of return outputs is returned in +// `outputs`. `*outputs` is owned by and has the lifetime of `results`. +TF_CAPI_EXPORT extern void TF_ImportGraphDefResultsReturnOutputs( + TF_ImportGraphDefResults* results, int* num_outputs, TF_Output** outputs); + +// Fetches the return operations requested via +// TF_ImportGraphDefOptionsAddReturnOperation(). The number of fetched +// operations is returned in `num_opers`. The array of return operations is +// returned in `opers`. `*opers` is owned by and has the lifetime of `results`. +TF_CAPI_EXPORT extern void TF_ImportGraphDefResultsReturnOperations( + TF_ImportGraphDefResults* results, int* num_opers, TF_Operation*** opers); + +// Fetches any input mappings requested via +// TF_ImportGraphDefOptionsAddInputMapping() that weren't used as input to any +// node in the imported graph def. The number of fetched mappings is returned in +// `num_unused_input_mappings`. The array of each mapping's source node name is +// returned in `src_names`, and the array of each mapping's source index is +// returned in `src_indexes`. +// +// `*src_names`, `*src_indexes`, and the memory backing each string in +// `src_names` are owned by and have the lifetime of `results`. +TF_CAPI_EXPORT extern void TF_ImportGraphDefResultsUnusedInputMappings( + TF_ImportGraphDefResults* results, int* num_unused_input_mappings, + const char*** src_names, int** src_indexes); + +// Deletes a results object returned by TF_GraphImportGraphDefWithResults(). +TF_CAPI_EXPORT extern void TF_DeleteImportGraphDefResults( + TF_ImportGraphDefResults* results); + +// Import the graph serialized in `graph_def` into `graph`. Returns nullptr and +// a bad status on error. Otherwise, returns a populated +// TF_ImportGraphDefResults instance. The returned instance must be deleted via +// TF_DeleteImportGraphDefResults(). +TF_CAPI_EXPORT extern TF_ImportGraphDefResults* +TF_GraphImportGraphDefWithResults(TF_Graph* graph, const TF_Buffer* graph_def, + const TF_ImportGraphDefOptions* options, + TF_Status* status); + // Import the graph serialized in `graph_def` into `graph`. +// Convenience function for when only return outputs are needed. // // `num_return_outputs` must be the number of return outputs added (i.e. the // result of TF_ImportGraphDefOptionsNumReturnOutputs()). If @@ -926,7 +981,7 @@ TF_CAPI_EXPORT extern void TF_GraphImportGraphDefWithReturnOutputs( int num_return_outputs, TF_Status* status); // Import the graph serialized in `graph_def` into `graph`. -// Convenience function for when no return outputs have been added. +// Convenience function for when no results are needed. TF_CAPI_EXPORT extern void TF_GraphImportGraphDef( TF_Graph* graph, const TF_Buffer* graph_def, const TF_ImportGraphDefOptions* options, TF_Status* status); diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h index 23ec1fac6f..bb04e01bee 100644 --- a/tensorflow/c/c_api_internal.h +++ b/tensorflow/c/c_api_internal.h @@ -18,7 +18,9 @@ limitations under the License. #include "tensorflow/c/c_api.h" +#include #include +#include #include #include @@ -124,6 +126,20 @@ struct TF_Session { struct TF_ImportGraphDefOptions { tensorflow::ImportGraphDefOptions opts; + + // Backing memory for TensorId fields in opts. + // TODO(skyewm): it'd be better if ImportGraphDefOptions owned this. + std::list tensor_id_data; +}; + +struct TF_ImportGraphDefResults { + std::vector return_tensors; + std::vector return_nodes; + std::vector unused_key_names; + std::vector unused_key_indexes; + + // Backing memory for unused_key_names values. + std::list unused_key_names_data; }; struct TF_DeviceList { diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc index d220bc5e95..05881e619b 100644 --- a/tensorflow/c/c_api_test.cc +++ b/tensorflow/c/c_api_test.cc @@ -573,7 +573,7 @@ TEST(CAPI, ImportGraphDef) { TF_GraphToGraphDef(graph, graph_def, s); ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - // Import it again, with a prefix, in a fresh graph. + // Import it, with a prefix, in a fresh graph. TF_DeleteGraph(graph); graph = TF_NewGraph(); TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions(); @@ -588,8 +588,8 @@ TEST(CAPI, ImportGraphDef) { ASSERT_TRUE(feed != nullptr); ASSERT_TRUE(neg != nullptr); - // Import it again, with an input mapping and return outputs, into the same - // graph. + // Import it again, with an input mapping, return outputs, and a return + // operation, into the same graph. TF_DeleteImportGraphDefOptions(opts); opts = TF_NewImportGraphDefOptions(); TF_ImportGraphDefOptionsSetPrefix(opts, "imported2"); @@ -597,9 +597,10 @@ TEST(CAPI, ImportGraphDef) { TF_ImportGraphDefOptionsAddReturnOutput(opts, "feed", 0); TF_ImportGraphDefOptionsAddReturnOutput(opts, "scalar", 0); EXPECT_EQ(2, TF_ImportGraphDefOptionsNumReturnOutputs(opts)); - TF_Output return_outputs[2]; - TF_GraphImportGraphDefWithReturnOutputs(graph, graph_def, opts, - return_outputs, 2, s); + TF_ImportGraphDefOptionsAddReturnOperation(opts, "scalar"); + EXPECT_EQ(1, TF_ImportGraphDefOptionsNumReturnOperations(opts)); + TF_ImportGraphDefResults* results = + TF_GraphImportGraphDefWithResults(graph, graph_def, opts, s); ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); TF_Operation* scalar2 = TF_GraphOperationByName(graph, "imported2/scalar"); @@ -615,11 +616,26 @@ TEST(CAPI, ImportGraphDef) { EXPECT_EQ(0, neg_input.index); // Check return outputs + TF_Output* return_outputs; + int num_return_outputs; + TF_ImportGraphDefResultsReturnOutputs(results, &num_return_outputs, + &return_outputs); + ASSERT_EQ(2, num_return_outputs); EXPECT_EQ(feed2, return_outputs[0].oper); EXPECT_EQ(0, return_outputs[0].index); EXPECT_EQ(scalar, return_outputs[1].oper); // remapped EXPECT_EQ(0, return_outputs[1].index); + // Check return operation + TF_Operation** return_opers; + int num_return_opers; + TF_ImportGraphDefResultsReturnOperations(results, &num_return_opers, + &return_opers); + ASSERT_EQ(1, num_return_opers); + EXPECT_EQ(scalar2, return_opers[0]); // not remapped + + TF_DeleteImportGraphDefResults(results); + // Import again, with control dependencies, into the same graph. TF_DeleteImportGraphDefOptions(opts); opts = TF_NewImportGraphDefOptions(); @@ -689,6 +705,113 @@ TEST(CAPI, ImportGraphDef) { TF_DeleteStatus(s); } +TEST(CAPI, ImportGraphDef_WithReturnOutputs) { + TF_Status* s = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + + // Create a graph with two nodes: x and 3 + Placeholder(graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "feed") != nullptr); + TF_Operation* oper = ScalarConst(3, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "scalar") != nullptr); + Neg(oper, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "neg") != nullptr); + + // Export to a GraphDef. + TF_Buffer* graph_def = TF_NewBuffer(); + TF_GraphToGraphDef(graph, graph_def, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + // Import it in a fresh graph with return outputs. + TF_DeleteGraph(graph); + graph = TF_NewGraph(); + TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions(); + TF_ImportGraphDefOptionsAddReturnOutput(opts, "feed", 0); + TF_ImportGraphDefOptionsAddReturnOutput(opts, "scalar", 0); + EXPECT_EQ(2, TF_ImportGraphDefOptionsNumReturnOutputs(opts)); + TF_Output return_outputs[2]; + TF_GraphImportGraphDefWithReturnOutputs(graph, graph_def, opts, + return_outputs, 2, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + TF_Operation* scalar = TF_GraphOperationByName(graph, "scalar"); + TF_Operation* feed = TF_GraphOperationByName(graph, "feed"); + TF_Operation* neg = TF_GraphOperationByName(graph, "neg"); + ASSERT_TRUE(scalar != nullptr); + ASSERT_TRUE(feed != nullptr); + ASSERT_TRUE(neg != nullptr); + + // Check return outputs + EXPECT_EQ(feed, return_outputs[0].oper); + EXPECT_EQ(0, return_outputs[0].index); + EXPECT_EQ(scalar, return_outputs[1].oper); + EXPECT_EQ(0, return_outputs[1].index); + + TF_DeleteImportGraphDefOptions(opts); + TF_DeleteBuffer(graph_def); + TF_DeleteGraph(graph); + TF_DeleteStatus(s); +} + +TEST(CAPI, ImportGraphDef_UnusedInputMappings) { + TF_Status* s = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + + // Create a graph with two nodes: x and 3 + Placeholder(graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "feed") != nullptr); + TF_Operation* oper = ScalarConst(3, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "scalar") != nullptr); + Neg(oper, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "neg") != nullptr); + + // Export to a GraphDef. + TF_Buffer* graph_def = TF_NewBuffer(); + TF_GraphToGraphDef(graph, graph_def, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + // Import it in a fresh graph. + TF_DeleteGraph(graph); + graph = TF_NewGraph(); + TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions(); + TF_GraphImportGraphDef(graph, graph_def, opts, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + TF_Operation* scalar = TF_GraphOperationByName(graph, "scalar"); + + // Import it in a fresh graph with an unused input mapping. + TF_DeleteImportGraphDefOptions(opts); + opts = TF_NewImportGraphDefOptions(); + TF_ImportGraphDefOptionsSetPrefix(opts, "imported"); + TF_ImportGraphDefOptionsAddInputMapping(opts, "scalar", 0, {scalar, 0}); + TF_ImportGraphDefOptionsAddInputMapping(opts, "fake", 0, {scalar, 0}); + TF_ImportGraphDefResults* results = + TF_GraphImportGraphDefWithResults(graph, graph_def, opts, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + // Check unused input mappings + int num_unused_input_mappings; + const char** src_names; + int* src_indexes; + TF_ImportGraphDefResultsUnusedInputMappings( + results, &num_unused_input_mappings, &src_names, &src_indexes); + ASSERT_EQ(1, num_unused_input_mappings); + EXPECT_EQ(string("fake"), string(src_names[0])); + EXPECT_EQ(0, src_indexes[0]); + + TF_DeleteImportGraphDefResults(results); + TF_DeleteImportGraphDefOptions(opts); + TF_DeleteBuffer(graph_def); + TF_DeleteGraph(graph); + TF_DeleteStatus(s); +} + TEST(CAPI, Session) { TF_Status* s = TF_NewStatus(); TF_Graph* graph = TF_NewGraph(); diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index b2c193b050..9432775ff3 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -90,7 +90,7 @@ class GraphConstructor { bool skip_mapped_nodes; std::vector control_dependencies; std::vector return_tensors; - std::vector return_nodes; + std::vector return_nodes; // TODO(ashankar): This bool exists to separate out functionality required // to make ImportGraphDef a close equivalent of Python's import_graph_def diff --git a/tensorflow/core/graph/graph_constructor.h b/tensorflow/core/graph/graph_constructor.h index 6cd9347d96..a364478878 100644 --- a/tensorflow/core/graph/graph_constructor.h +++ b/tensorflow/core/graph/graph_constructor.h @@ -110,7 +110,7 @@ struct ImportGraphDefOptions { // Unlike `return_tensors`, `input_map` has no effect on the nodes // returned. `return_nodes` must be empty if `skip_mapped_nodes` is true. // TODO(skyewm): make this work with `skip_mapped_nodes` if there's a need. - std::vector return_nodes; + std::vector return_nodes; // TODO(ashankar): Enable handling of GraphDefs produced by newer binaries // with ops that are not defined in the binary calling ImportGraphDef. -- GitLab From b73743e3a035c4da7fd6e223e53fe9d817c04cc4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 08:48:05 -0700 Subject: [PATCH 507/573] Remove accidental disablation of (already manual) tests. PiperOrigin-RevId: 173898910 --- tensorflow/compiler/tests/randomized_tests.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc index c8a32f9e29..6a8c3bcd55 100644 --- a/tensorflow/compiler/tests/randomized_tests.cc +++ b/tensorflow/compiler/tests/randomized_tests.cc @@ -1551,7 +1551,6 @@ TEST_F(OpTest, DepthToSpace) { } TEST_F(OpTest, DepthwiseConv2DNative) { - if (1) return; Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); std::uniform_int_distribution random_int(1, 5); @@ -1575,7 +1574,6 @@ TEST_F(OpTest, DepthwiseConv2DNative) { } TEST_F(OpTest, DepthwiseConv2DBackpropFilter) { - if (1) return; Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); std::uniform_int_distribution random_int(1, 5); @@ -1603,7 +1601,6 @@ TEST_F(OpTest, DepthwiseConv2DBackpropFilter) { } TEST_F(OpTest, DepthwiseConv2DBackpropInput) { - if (1) return; Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); std::uniform_int_distribution random_int(1, 5); @@ -1631,7 +1628,6 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) { } TEST_F(OpTest, Diag) { - if (1) return; Repeatedly([this]() { auto type = Choose(kAllXlaTypes); std::vector dims; -- GitLab From 494672475bd9b36b36460b4760997d929a65f823 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 08:52:31 -0700 Subject: [PATCH 508/573] Added "NOTE: You may only install TensorFlow on 64-bit machines" to all the TensorFlow Install guides. PiperOrigin-RevId: 173899394 --- tensorflow/docs_src/install/install_c.md | 17 ++++++---- tensorflow/docs_src/install/install_go.md | 17 ++++++---- tensorflow/docs_src/install/install_java.md | 26 +++++++------- tensorflow/docs_src/install/install_linux.md | 34 +++++++++++-------- tensorflow/docs_src/install/install_mac.md | 7 +++- .../docs_src/install/install_sources.md | 34 +++++++++++-------- .../docs_src/install/install_windows.md | 9 ++++- 7 files changed, 87 insertions(+), 57 deletions(-) diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 586bb6dead..70f756b194 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -9,10 +9,13 @@ The API leans towards simplicity and uniformity rather than convenience. ## Supported Platforms -You may install TensorFlow for C on the following operating systems: +This guide explains how to install TensorFlow for C. Although these +instructions might also work on other variants, we have only tested +(and we only support) these instructions on machines meeting the +following requirements: - * Linux - * Mac OS X + * Linux, 64-bit, x86 + * macOS X, Version 10.11 (El Capitan) or higher ## Installation @@ -26,13 +29,13 @@ enable TensorFlow for C: following guides: * @{$install_linux#determine_which_tensorflow_to_install$Installing TensorFlow on Linux} - * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on Mac OS} + * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on macOS} 2. Download and extract the TensorFlow C library into `/usr/local/lib` by invoking the following shell commands: TF_TYPE="cpu" # Change to "gpu" for GPU support - OS="linux" # Change to "darwin" for Mac OS + OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" | @@ -57,9 +60,9 @@ enable TensorFlow for C: directory (for example, `~/mydir/lib`) to two environment variables. For example: - 
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and Mac OS X
+     
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and macOS X
      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/mydir/lib # For Linux only
-     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For Mac OS X only

+     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For macOS X only
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 1d00661d83..eca2ecc5ac 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -5,16 +5,19 @@ well-suited to loading models created in Python and executing them within a Go application. This guide explains how to install and set up the [TensorFlow Go package](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go). -**WARNING:** The TensorFlow Go API is *not* covered by the TensorFlow +Warning: The TensorFlow Go API is *not* covered by the TensorFlow [API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). ## Supported Platforms -You may install TensorFlow for Go on the following operating systems: +This guide explains how to install TensorFlow for Go. Although these +instructions might also work on other variants, we have only tested +(and we only support) these instructions on machines meeting the +following requirements: - * Linux - * Mac OS X + * Linux, 64-bit, x86 + * macOS X, 10.11 (El Capitan) or higher ## Installation @@ -27,7 +30,7 @@ steps to install this library and enable TensorFlow for Go: "Determine which TensorFlow to install" in one of the following guides: * @{$install_linux#determine_which_tensorflow_to_install$Installing TensorFlow on Linux} - * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on Mac OS} + * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on macOS} 2. Download and extract the TensorFlow C library into `/usr/local/lib` by invoking the following shell commands: @@ -57,9 +60,9 @@ steps to install this library and enable TensorFlow for Go: directory (for example, `~/mydir/lib`) to two environment variables as follows: - 
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and Mac OS X
+     
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and macOS X
      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/mydir/lib # For Linux only
-     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For Mac OS X only

+     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For macOS X only
4. Now that the TensorFlow C library is installed, invoke `go get` as follows to download the appropriate packages and their dependencies: diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index 3b3acfdcb3..8eaec3712a 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -6,18 +6,20 @@ Java application. This guide explains how to install [TensorFlow for Java](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary) and use it in a Java application. -**WARNING:** The TensorFlow Java API is *not* covered by the TensorFlow +Warning: The TensorFlow Java API is *not* covered by the TensorFlow [API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). ## Supported Platforms -TensorFlow for Java is supported on the following operating systems: +This guide explains how to install TensorFlow for Java. Although these +instructions might also work on other variants, we have only tested +(and we only support) these instructions on machines meeting the +following requirements: - * Linux - * Mac OS X - * Windows - * Android + * Ubuntu 14.04 or higher; 64-bit, x86 + * macOS X 10.11 (El Capitan) or higher + * Windows 7 or higher; 64-bit, x86 The installation instructions for Android are in a separate [Android TensorFlow Support page](https://www.tensorflow.org/code/tensorflow/contrib/android). @@ -81,14 +83,14 @@ As an example, these steps will create a Maven project that uses TensorFlow: public static void main(String[] args) throws Exception { try (Graph g = new Graph()) { final String value = "Hello from " + TensorFlow.version(); - + // Construct the computation graph with a single operation, a constant // named "MyConst" with a value "value". try (Tensor t = Tensor.create(value.getBytes("UTF-8"))) { // The Java API doesn't yet include convenience functions for adding operations. g.opBuilder("Const", "MyConst").setAttr("dtype", t.dataType()).setAttr("value", t).build(); } - + // Execute the "MyConst" operation in a Session. try (Session s = new Session(g); Tensor output = s.runner().fetch("MyConst").run().get(0)) { @@ -117,9 +119,9 @@ This section describes how to use TensorFlow using the `java` and `javac` commands from a JDK installation. If your project uses Apache Maven, then refer to the simpler instructions above instead. -### Install on Linux or Mac OS +### Install on Linux or macOS -Take the following steps to install TensorFlow for Java on Linux or Mac OS: +Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar), @@ -130,7 +132,7 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS: "Determine which TensorFlow to install" in one of the following guides: * @{$install_linux#determine_which_tensorflow_to_install$Installing TensorFlow on Linux} - * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on Mac OS} + * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on macOS} 3. Download and extract the appropriate Java Native Interface (JNI) file for your operating system and processor support by running the @@ -212,7 +214,7 @@ two files are available to the JVM: * the extracted JNI library For example, the following command line executes the `HelloTF` program on Linux -and Mac OS X: +and macOS X: 
java -cp libtensorflow-1.4.0-rc0.jar:. -Djava.library.path=./jni HelloTF
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 9d204cc246..2b321e7dcb 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -1,8 +1,12 @@ # Installing TensorFlow on Ubuntu -This guide explains how to install TensorFlow on Ubuntu. These instructions -might also work on other Linux variants, but we have only tested (and we -only support) these instructions on Ubuntu 14.04 or higher. +This guide explains how to install TensorFlow on Ubuntu. Although these +instructions might also work on other Linux variants, we have only +tested (and we only support) these instructions on machines meeting the +following requirements: + + * 64-bit desktops or laptops + * Ubuntu 14.04 or higher ## Determine which TensorFlow to install @@ -128,12 +132,12 @@ Take the following steps to install TensorFlow with Virtualenv: 1. Install pip and virtualenv by issuing one of the following commands: 
$ sudo apt-get install python-pip python-dev python-virtualenv # for Python 2.7
-     $ sudo apt-get install python3-pip python3-dev python-virtualenv # for Python 3.n
+ $ sudo apt-get install python3-pip python3-dev python-virtualenv # for Python 3.n 2. Create a virtualenv environment by issuing one of the following commands: 
$ virtualenv --system-site-packages targetDirectory # for Python 2.7
-     $ virtualenv --system-site-packages -p python3 targetDirectory # for Python 3.n
+ $ virtualenv --system-site-packages -p python3 targetDirectory # for Python 3.n where targetDirectory specifies the top of the virtualenv tree. Our instructions assume that @@ -144,7 +148,7 @@ Take the following steps to install TensorFlow with Virtualenv: commands: 
$ source ~/tensorflow/bin/activate # bash, sh, ksh, or zsh
-     $ source ~/tensorflow/bin/activate.csh  # csh or tcsh
+ $ source ~/tensorflow/bin/activate.csh # csh or tcsh The preceding source command should change your prompt to the following: @@ -159,9 +163,9 @@ Take the following steps to install TensorFlow with Virtualenv: virtualenv environment: 
(tensorflow)$ pip install --upgrade tensorflow      # for Python 2.7
-     (tensorflow)$ pip3 install --upgrade tensorflow     # for Python 3.n
-     (tensorflow)$ pip install --upgrade tensorflow-gpu  # for Python 2.7 and GPU
-     (tensorflow)$ pip3 install --upgrade tensorflow-gpu # for Python 3.n and GPU
+ (tensorflow)$ pip3 install --upgrade tensorflow # for Python 3.n + (tensorflow)$ pip install --upgrade tensorflow-gpu # for Python 2.7 and GPU + (tensorflow)$ pip3 install --upgrade tensorflow-gpu # for Python 3.n and GPU If the preceding command succeeds, skip Step 6. If the preceding command fails, perform Step 6. @@ -171,7 +175,7 @@ Take the following steps to install TensorFlow with Virtualenv: by issuing a command of the following format: 
(tensorflow)$ pip install --upgrade tfBinaryURL   # Python 2.7
-     (tensorflow)$ pip3 install --upgrade tfBinaryURL  # Python 3.n
+ (tensorflow)$ pip3 install --upgrade tfBinaryURL # Python 3.n where tfBinaryURL identifies the URL of the TensorFlow Python package. The appropriate value of @@ -199,7 +203,7 @@ Note that you must activate the virtualenv environment each time you use TensorFlow. If the virtualenv environment is not currently active, invoke one of the following commands: -
$ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
+
 $ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
 $ source ~/tensorflow/bin/activate.csh  # csh or tcsh

 
 When the virtualenv environment is active, you may run
@@ -265,9 +269,9 @@ take the following steps:
   1. Install TensorFlow by invoking **one** of the following commands:
 
      
$ pip install tensorflow      # Python 2.7; CPU support (no GPU support)
-     $ pip3 install tensorflow     # Python 3.n; CPU support (no GPU support)
-     $ pip install tensorflow-gpu  # Python 2.7;  GPU support
-     $ pip3 install tensorflow-gpu # Python 3.n; GPU support 

+    $ pip3 install tensorflow     # Python 3.n; CPU support (no GPU support)
+    $ pip install tensorflow-gpu  # Python 2.7;  GPU support
+    $ pip3 install tensorflow-gpu # Python 3.n; GPU support
If the preceding command runs to completion, you should now [validate your installation](#ValidateYourInstallation). @@ -276,7 +280,7 @@ take the following steps: by issuing a command of the following format: 
$ sudo pip  install --upgrade tfBinaryURL   # Python 2.7
-     $ sudo pip3 install --upgrade tfBinaryURL   # Python 3.n
+ $ sudo pip3 install --upgrade tfBinaryURL # Python 3.n where tfBinaryURL identifies the URL of the TensorFlow Python package. The appropriate value of diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index 6da22784bf..d799298b8b 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -1,6 +1,11 @@ # Installing TensorFlow on macOS -This guide explains how to install TensorFlow on macOS. +This guide explains how to install TensorFlow on macOS. Although these +instructions might also work on other macOS variants, we have only +tested (and we only support) these instructions on machines meeting the +following requirements: + + * macOS X 10.11 (El Capitan) or higher Note: As of version 1.2, TensorFlow no longer provides GPU support on macOS. diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index b853d87816..28bc5f5159 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -2,7 +2,7 @@ This guide explains how to build TensorFlow sources into a TensorFlow binary and how to install that TensorFlow binary. Note that we provide -well-tested, pre-built TensorFlow binaries for Linux, Mac, and Windows +well-tested, pre-built TensorFlow binaries for Ubuntu, macOS, and Windows systems. In addition, there are pre-built TensorFlow [docker images](https://hub.docker.com/r/tensorflow/tensorflow/). So, don't build a TensorFlow binary yourself unless you are very @@ -10,16 +10,22 @@ comfortable building complex packages from source and dealing with the inevitable aftermath should things not go exactly as documented. If the last paragraph didn't scare you off, welcome. This guide explains -how to build TensorFlow on the following operating systems: +how to build TensorFlow on 64-bit desktops and laptops running either of +the following operating systems: * Ubuntu -* Mac OS X +* macOS X -We don't officially support building TensorFlow on Windows; however, you may try -to build TensorFlow on Windows if you don't mind using the highly experimental -[Bazel on Windows](https://bazel.build/versions/master/docs/windows.html) -or -[TensorFlow CMake build](https://github.com/tensorflow/tensorflow/tree/r0.12/tensorflow/contrib/cmake). +Note: Some users have successfully built and installed TensorFlow from +sources on non-supported systems. Please remember that we do not fix +issues stemming from these attempts. + +We **do not support** building TensorFlow on Windows. That said, if you'd +like to try to build TensorFlow on Windows anyway, use either of the +following: + +* [Bazel on Windows](https://bazel.build/versions/master/docs/windows.html) +* [TensorFlow CMake build](https://github.com/tensorflow/tensorflow/tree/r0.12/tensorflow/contrib/cmake) ## Determine which TensorFlow to install @@ -40,7 +46,7 @@ install: software requirements described in one of the following documents: * @{$install_linux#NVIDIARequirements$Installing TensorFlow on Ubuntu} - * @{$install_mac#NVIDIARequirements$Installing TensorFlow on Mac OS} + * @{$install_mac#NVIDIARequirements$Installing TensorFlow on macOS} ## Clone the TensorFlow repository @@ -70,7 +76,7 @@ issue the following command: Next, you must prepare your environment for [Linux](#PrepareLinux) or -[Mac OS](#PrepareMac) +[macOS](#PrepareMac) @@ -157,7 +163,7 @@ After preparing the environment, you must now -## Prepare environment for Mac OS +## Prepare environment for macOS Before building TensorFlow, you must install the following on your system: @@ -238,8 +244,8 @@ One of the questions that `configure` will ask is as follows: Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -march=native] -This question refers to a later phase in which you'll use bazel to -[build the pip package](#build-the-pip-package). We recommend +This question refers to a later phase in which you'll use bazel to +[build the pip package](#build-the-pip-package). We recommend accepting the default (`-march=native`), which will optimize the generated code for your local machine's CPU type. However, if you are building TensorFlow on one CPU type but will run TensorFlow on @@ -288,7 +294,7 @@ Please specify a list of comma-separated Cuda compute capabilities you want to b You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus. Please note that each additional compute capability significantly increases your build time and binary size. [Default is: "3.5,5.2"]: 3.0 -Do you wish to build TensorFlow with MPI support? [y/N] +Do you wish to build TensorFlow with MPI support? [y/N] MPI support will not be enabled for TensorFlow Configuration finished diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index f0d580d803..4098ee5b2e 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -1,6 +1,13 @@ # Installing TensorFlow on Windows -This guide explains how to install TensorFlow on Windows. +This guide explains how to install TensorFlow on Windows. Although these +instructions might also work on other Windows variants, we have only +tested (and we only support) these instructions on machines meeting the +following requirements: + + * 64-bit, x86 desktops or laptops + * Windows 7 or later + ## Determine which TensorFlow to install -- GitLab From 4723f8f6ed4e43632ea90456bd36a1f8e8b1aeb8 Mon Sep 17 00:00:00 2001 From: RJ Ryan Date: Mon, 30 Oct 2017 09:11:59 -0700 Subject: [PATCH 509/573] Support SymbolicGradient for functions with non-trainable arguments. The non-trainable arguments end up with None as their incoming out_grad, which is not a valid input to SymbolicGradient (inputs have to be convertible to Tensor, and None isn't). PiperOrigin-RevId: 173901727 --- tensorflow/python/framework/function_test.py | 18 ++++++++++++++++++ tensorflow/python/ops/gradients_impl.py | 14 ++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index fbc1045b5b..36b0737cfc 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -864,6 +864,24 @@ class FunctionTest(test.TestCase): [result]) self.assertEqual(len(f.signature.input_arg), 3) + def testGradientWithIntegerFunctionArgument(self): + @function.Defun(dtypes.int32, dtypes.float32) + def Foo(t, x): + return x[t] + + g = ops.Graph() + with g.as_default(): + inp = array_ops.placeholder(dtypes.float32) + t = constant_op.constant(0, dtypes.int32) + out = Foo(t, inp) + dinp, = gradients_impl.gradients(out, [inp]) + + x = np.zeros((2,)).astype(np.float32) + with session.Session(graph=g) as sess: + self.assertAllClose( + np.array([1.0, 0.0]).astype(np.float32), + sess.run(dinp, {inp: x})) + @test_util.with_c_api class FunctionsFromProtos(test.TestCase): diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py index eb34a35a2b..97a3486f61 100644 --- a/tensorflow/python/ops/gradients_impl.py +++ b/tensorflow/python/ops/gradients_impl.py @@ -582,8 +582,10 @@ def gradients(ys, # therefore dC/doutput[i] is 0. for i, out_grad in enumerate(out_grads): if (not isinstance(out_grad, ops.Tensor) and - not out_grad) and _IsTrainable(op.outputs[i]): - # Only floating-point outputs get a zero gradient. Gradient + not out_grad) and ((not grad_fn and is_func_call) or + _IsTrainable(op.outputs[i])): + # Only trainable outputs or outputs for a function call that + # will use SymbolicGradient get a zero gradient. Gradient # functions should ignore the gradient for other outputs. # TODO(apassos) gradients of resource handles might be an # issue here because of zeros. @@ -670,15 +672,15 @@ def _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state): grad_state.pending_exits_count -= 1 if grad_state.pending_exits_count == 0: # We now have all the exits so process them. - has_real_grad = False + has_not_none_grad = False for y in grad_state.deferred_exits: if _HasAnyNotNoneGrads(grads, y.op): - has_real_grad = True + has_not_none_grad = True queue.append(y.op) else: grad_state.unused_exits.append(y) - if has_real_grad: - # For an unused exit, if it has floating-point outputs, backprop + if has_not_none_grad: + # For an unused exit, if it has trainable outputs, backprop # a zero gradient. Otherwise, just ignore it. for y in grad_state.unused_exits: if _IsTrainable(y): -- GitLab From 7fd261602677d3c251fba05264a20318231deb76 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 30 Oct 2017 09:20:20 -0700 Subject: [PATCH 510/573] Add TF_GraphVersions() to C API and use in Graph.graph_def_versions() PiperOrigin-RevId: 173902666 --- tensorflow/c/c_api.cc | 11 +++++++++++ tensorflow/c/c_api.h | 5 +++++ tensorflow/python/framework/ops.py | 11 ++++++++++- tensorflow/python/framework/ops_test.py | 13 ++++++------- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index b43d202f4e..6dd1b99910 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -81,6 +81,7 @@ using tensorflow::TensorBuffer; using tensorflow::TensorId; using tensorflow::TensorShape; using tensorflow::TensorShapeProto; +using tensorflow::VersionDef; using tensorflow::error::Code; using tensorflow::errors::FailedPrecondition; using tensorflow::errors::InvalidArgument; @@ -1809,6 +1810,16 @@ void TF_GraphGetOpDef(TF_Graph* graph, const char* op_name, status->status = MessageToBuffer(*op_def, output_op_def); } +void TF_GraphVersions(TF_Graph* graph, TF_Buffer* output_version_def, + TF_Status* status) { + VersionDef versions; + { + mutex_lock l(graph->mu); + versions = graph->graph.versions(); + } + status->status = MessageToBuffer(versions, output_version_def); +} + TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions() { return new TF_ImportGraphDefOptions; } diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index ca5c934634..bb569d67fc 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -871,6 +871,11 @@ TF_CAPI_EXPORT extern void TF_GraphGetOpDef(TF_Graph* graph, TF_Buffer* output_op_def, TF_Status* status); +// Returns the serialized VersionDef proto for this graph. +TF_CAPI_EXPORT extern void TF_GraphVersions(TF_Graph* graph, + TF_Buffer* output_version_def, + TF_Status* status); + // TF_ImportGraphDefOptions holds options that can be passed to // TF_GraphImportGraphDef. typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions; diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 63f70a1a9d..b5e3e548bd 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -2713,7 +2713,16 @@ class Graph(object): A `VersionDef`. """ # pylint: enable=line-too-long - return self._graph_def_versions + if self._c_graph: + with errors.raise_exception_on_not_ok_status() as status: + with c_api_util.tf_buffer() as buf: + c_api.TF_GraphVersions(self._c_graph, buf, status) + data = c_api.TF_GetBuffer(buf) + version_def = versions_pb2.VersionDef() + version_def.ParseFromString(compat.as_bytes(data)) + return version_def + else: + return self._graph_def_versions @property def seed(self): diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index 59c0288457..b1269b84bd 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -1642,17 +1642,16 @@ class KernelLabelTest(test_util.TensorFlowTestCase): self.assertAllEqual(b"My label is: overload_2", overload_2.eval()) +@test_util.with_c_api class AsGraphDefTest(test_util.TensorFlowTestCase): def testGraphDefVersion(self): """Test that the graphdef version is plumbed through to kernels.""" - for version in range(versions.GRAPH_DEF_VERSION_MIN_PRODUCER, - versions.GRAPH_DEF_VERSION + 2): - with ops.Graph().as_default() as g: - g.graph_def_versions.producer = version - with self.test_session(graph=g): - v = test_ops.graph_def_version().eval() - self.assertEqual(version, v) + with ops.Graph().as_default() as g: + version = g.graph_def_versions.producer + with self.test_session(graph=g): + v = test_ops.graph_def_version().eval() + self.assertEqual(version, v) def testAddShapes(self): with ops.Graph().as_default() as g: -- GitLab From 85f8d924086657852c900c0ba7e8f0fbdac0a509 Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Mon, 30 Oct 2017 09:31:50 -0700 Subject: [PATCH 511/573] [tensorflow training input] If SparseTensors are used in batch* ops, ensure restoration. This forces the ST restore op to be called if any tensors are accessed at the output of the batch, thus fixing a memory leak. Solution suggested by Derek Murray. Fixes #13999. PiperOrigin-RevId: 173904309 --- tensorflow/python/training/input.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index f645d8cf39..331a51e8bc 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -574,7 +574,23 @@ def _restore_sparse_tensors(stored_list, sparse_info_list): rank=(info.rank + 1).value) if info.sparse else s for (s, info) in zip(stored_list, sparse_info_list)] - return tensors if received_sequence else tensors[0] + has_st = any(isinstance(x, sparse_tensor.SparseTensor) for x in tensors) + if has_st: + t_values = [ + x.values if isinstance(x, sparse_tensor.SparseTensor) + else x + for x in tensors] + with_deps = lambda x: control_flow_ops.with_dependencies(t_values, x) + ensure_restore_tensors = [ + sparse_tensor.SparseTensor(indices=with_deps(x.indices), + values=with_deps(x.values), + dense_shape=with_deps(x.dense_shape)) + if isinstance(x, sparse_tensor.SparseTensor) + else with_deps(x) + for x in tensors] + else: + ensure_restore_tensors = tensors + return ensure_restore_tensors if received_sequence else tensors[0] def _validate(tensor_list): -- GitLab From e8ac0b48f443879d9e3d516b0b3a151978128423 Mon Sep 17 00:00:00 2001 From: Akshay Agrawal Date: Mon, 30 Oct 2017 10:44:06 -0700 Subject: [PATCH 512/573] Report a nicer error message when differentiating a function that returns None in eager PiperOrigin-RevId: 173914883 --- tensorflow/python/eager/backprop.py | 13 +++++++++++++ tensorflow/python/eager/backprop_test.py | 21 +++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index be733405a3..6f7f2117be 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -332,6 +332,9 @@ def implicit_val_and_grad(f): A function which, when called, returns a tuple pair. Its first element is the value to which the function evaluates. Its second element is list of (gradient, variable) pairs. + + Raises: + ValueError: if `f` returns None. """ # TODO(cais): Remove calls to tf.constant() once the gradients functions # accept lists and np.ndarrays. @@ -341,6 +344,10 @@ def implicit_val_and_grad(f): tape.push_new_tape() try: end_node = f(*args) + if end_node is None: + raise ValueError("Cannot differentiate a function that returns None; " + "did you forget to return a value from {}?".format( + f.__name__)) variables = tape.top_tape_watched_variables() finally: popped_tape = tape.pop_tape() @@ -630,6 +637,8 @@ def make_vjp(f, params=None): # result is 9.0 vjp() # the vjp function rturns 6.0 + Raises: + ValueError: if `f` returns None. """ def decorated(*args, **kwds): @@ -649,6 +658,10 @@ def make_vjp(f, params=None): sources.append(args[i]) tape.watch(args[i]) result = f(*args) + if result is None: + raise ValueError("Cannot differentiate a function that returns None; " + "did you forget to return a value from {}?".format( + f.__name__)) flat_result = nest.flatten(result) flat_result = [gen_array_ops.identity(x) for x in flat_result] result = nest.pack_sequence_as(result, flat_result) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index cf736fcb13..ed54b8e12e 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -588,5 +588,26 @@ class BackpropTest(test.TestCase): self.assertAllEqual(backprop.gradients_function(my_identity)(1.0)[0], 2.0) + def testDifferentiatingFunctionThatReturnsNone(self): + + def fn(x, y): + result = x*y # pylint: disable=unused-variable + + x = constant_op.constant(1) + y = constant_op.constant(2) + + loss_grads_fn = backprop.implicit_val_and_grad(fn) + with self.assertRaisesRegexp( + ValueError, 'Cannot differentiate a function that returns None; ' + 'did you forget to return a value from fn?'): + loss_grads_fn(x, y) + + val_and_grads_fn = backprop.val_and_grad_function(fn) + with self.assertRaisesRegexp( + ValueError, 'Cannot differentiate a function that returns None; ' + 'did you forget to return a value from fn?'): + val_and_grads_fn(x, y) + + if __name__ == '__main__': test.main() -- GitLab From cef680b5320f85d155d6e16c607021e7182c5df6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 10:44:36 -0700 Subject: [PATCH 513/573] Enable shape inference on functions in grappler. PiperOrigin-RevId: 173914941 --- .../core/common_runtime/shape_refiner.h | 4 + tensorflow/core/graph/graph_constructor.cc | 7 +- .../core/grappler/costs/graph_properties.cc | 3 + .../grappler/costs/graph_properties_test.cc | 30 +++++ .../simple_function.pbtxt | 111 ++++++++++++++++++ 5 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt diff --git a/tensorflow/core/common_runtime/shape_refiner.h b/tensorflow/core/common_runtime/shape_refiner.h index d1288d671e..570b4db163 100644 --- a/tensorflow/core/common_runtime/shape_refiner.h +++ b/tensorflow/core/common_runtime/shape_refiner.h @@ -164,6 +164,10 @@ class ShapeRefiner { function_library_ = lib; } + bool function_shape_inference_supported() const { + return function_library_ != nullptr; + } + // Call this to keep nested shapes information for user-defined functions: // nested inferences will be available on the ExtendedInferenceContext for // each function node, forming a tree of shape inferences corresponding to the diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index 9432775ff3..8fe4f535fb 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -846,9 +846,10 @@ Status GraphConstructor::Convert() { } } - // TODO(skyewm): remove conditional when b/35715995 ("Functions lack shape - // inference") is resolved. - if (g_->flib_def().Find(node_def->name()) == nullptr) { + // Function shape inference is supported on an opt-in basis per + // ShapeRefiner. + if (refiner_->function_shape_inference_supported() || + g_->flib_def().Find(node_def->name()) == nullptr) { TF_RETURN_IF_ERROR(ValidateShape(node)); } diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index f62a21ace5..e9cb2ee09d 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -195,9 +195,12 @@ Status GraphProperties::RelaxEnqueueShapesAndMergeTypes( Status GraphProperties::InferStatically() { Graph graph(OpRegistry::Global()); + FunctionLibraryDefinition function_library(graph.op_registry(), + item_.graph.library()); ShapeRefiner shape_refiner(graph.versions(), graph.op_registry()); shape_refiner.set_require_shape_inference_fns(false); shape_refiner.set_disable_constant_propagation(true); + shape_refiner.set_function_library_for_shape_inference(&function_library); ImportGraphDefOptions options; Status s = ImportGraphDef(options, item_.graph, &graph, &shape_refiner); TF_RETURN_IF_ERROR(s); diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc index 975ec31b14..134db5ec5a 100644 --- a/tensorflow/core/grappler/costs/graph_properties_test.cc +++ b/tensorflow/core/grappler/costs/graph_properties_test.cc @@ -703,6 +703,36 @@ TEST_F(GraphPropertiesTest, InferRestoreOpShape_WithTwoNodesShareSameOutput) { EXPECT_EQ("float: [128,256]", PropToString(prop)); } +TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) { + // Test graph produced in python using: + /* + @function.Defun(*[tf.float32] * 2, noinline=True) + def MyAdd(x, y): + return tf.add(x,y) + + with tf.Graph().as_default(): + x = tf.constant(2.0, shape=[1, 2], dtype=tf.float32) + y = tf.constant(2.0, shape=[1, 2], dtype=tf.float32) + z = MyAdd(x, y) + z = MyAdd(x, z) + */ + // Check that the shape of the second MyAdd node propagates + // correctly. + GrapplerItem item; + string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath, + "simple_function.pbtxt"); + TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); + GraphProperties properties(item); + TF_CHECK_OK(properties.InferStatically()); + const auto props = properties.GetOutputProperties("MyAdd_55e046a8_1"); + const OpInfo::TensorProperties& prop = props[0]; + EXPECT_EQ(DT_FLOAT, prop.dtype()); + EXPECT_FALSE(prop.shape().unknown_rank()); + EXPECT_EQ(2, prop.shape().dim_size()); + EXPECT_EQ(1, prop.shape().dim(0).size()); + EXPECT_EQ(2, prop.shape().dim(1).size()); +} + } // namespace } // namespace grappler } // namespace tensorflow diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt new file mode 100644 index 0000000000..86b67f2049 --- /dev/null +++ b/tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt @@ -0,0 +1,111 @@ +node { + name: "Const" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 2 + } + } + float_val: 2.0 + } + } + } +} +node { + name: "Const_1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 2 + } + } + float_val: 2.0 + } + } + } +} +node { + name: "MyAdd_55e046a8" + op: "MyAdd_55e046a8" + input: "Const" + input: "Const_1" +} +node { + name: "MyAdd_55e046a8_1" + op: "MyAdd_55e046a8" + input: "Const" + input: "MyAdd_55e046a8" +} +library { + function { + signature { + name: "MyAdd_55e046a8" + input_arg { + name: "x" + type: DT_FLOAT + } + input_arg { + name: "y" + type: DT_FLOAT + } + output_arg { + name: "Add" + type: DT_FLOAT + } + } + node_def { + name: "Add" + op: "Add" + input: "x" + input: "y" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + } + ret { + key: "Add" + value: "Add:z:0" + } + attr { + key: "_noinline" + value { + b: true + } + } + } +} +versions { + producer: 24 + min_consumer: 12 +} -- GitLab From 89582677c3fd464a1e6cf94e39918a80f7bc6d77 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 30 Oct 2017 10:49:40 -0700 Subject: [PATCH 514/573] EagerVariableStore, for compatibility with functional layers. PiperOrigin-RevId: 173915730 --- tensorflow/contrib/eager/python/tfe.py | 2 + tensorflow/python/layers/convolutional.py | 48 -------------------- tensorflow/python/layers/core.py | 16 ------- tensorflow/python/layers/core_test.py | 18 ++++++++ tensorflow/python/layers/maxout.py | 8 ---- tensorflow/python/layers/normalization.py | 9 ---- tensorflow/python/layers/pooling.py | 48 -------------------- tensorflow/python/ops/variable_scope.py | 55 ++++++++++++++++++++++- 8 files changed, 73 insertions(+), 131 deletions(-) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index ab31893cd3..4164a815cd 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -52,6 +52,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@restore_variables_on_create @@Variable @@get_optimizer_variables +@@EagerVariableStore @@in_eager_mode @@in_graph_mode @@ -100,6 +101,7 @@ from tensorflow.python.framework.ops import eager_run as run from tensorflow.python.framework.test_util import IsolateTest from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable +from tensorflow.python.ops.variable_scope import EagerVariableStore from tensorflow.python.util.all_util import remove_undocumented defun = function.defun diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index c9bfafaee1..0c7ce02835 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -386,15 +386,7 @@ def conv1d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Conv1D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Conv1D instead.') layer = Conv1D( filters=filters, kernel_size=kernel_size, @@ -597,15 +589,7 @@ def conv2d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Conv2D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Conv2D instead.') layer = Conv2D( filters=filters, kernel_size=kernel_size, @@ -810,15 +794,7 @@ def conv3d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Conv3D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Conv3D instead.') layer = Conv3D( filters=filters, kernel_size=kernel_size, @@ -1140,15 +1116,7 @@ def separable_conv2d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.SeparableConv2d` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.SeparableConv2D instead.') layer = SeparableConv2D( filters=filters, kernel_size=kernel_size, @@ -1446,15 +1414,7 @@ def conv2d_transpose(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Conv2DTranspose` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Conv2DTranspose instead.') layer = Conv2DTranspose( filters=filters, kernel_size=kernel_size, @@ -1768,15 +1728,7 @@ def conv3d_transpose(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Conv3DTranspose` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Conv3DTranspose instead.') layer = Conv3DTranspose( filters=filters, kernel_size=kernel_size, diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index b30e5f2074..76e8fbef2f 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -234,15 +234,7 @@ def dense( Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Dense` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Dense instead.') layer = Dense(units, activation=activation, use_bias=use_bias, @@ -347,15 +339,7 @@ def dropout(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.Dropout` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.Dropout instead.') layer = Dropout(rate, noise_shape=noise_shape, seed=seed, name=name) return layer.apply(inputs, training=training) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index 5184b372ff..b67df89f81 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -23,6 +23,7 @@ import collections import numpy as np from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -258,6 +259,23 @@ class DenseTest(test.TestCase): self.assertAllClose(weights['scope/dense/bias'].read_value().eval(), np.zeros((2))) + def testEagerExecution(self): + with context.eager_mode(): + container = variable_scope.EagerVariableStore() + x = constant_op.constant([[2.0]]) + with container.as_default(): + y = core_layers.dense( + x, 1, name='my_dense', + kernel_initializer=init_ops.ones_initializer()) + self.assertAllEqual(y, [[2.0]]) + self.assertEqual(len(container.variables()), 2) + # Recreate the layer to test reuse. + with container.as_default(): + core_layers.dense( + x, 1, name='my_dense', + kernel_initializer=init_ops.ones_initializer()) + self.assertEqual(len(container.variables()), 2) + def testFunctionalDenseWithCustomGetter(self): called = [0] diff --git a/tensorflow/python/layers/maxout.py b/tensorflow/python/layers/maxout.py index 61cfd7f45c..ed048845a0 100644 --- a/tensorflow/python/layers/maxout.py +++ b/tensorflow/python/layers/maxout.py @@ -50,15 +50,7 @@ def maxout(inputs, num_units, axis=-1, name=None): Raises: ValueError: if num_units is not multiple of number of features. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.MaxOut` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'use tf.contrib.layers.MaxOut instead') return MaxOut(num_units=num_units, axis=axis, name=name)(inputs) diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index 4fbe4b574f..01f56abc70 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -729,16 +729,7 @@ def batch_normalization(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.BatchNormalization` - instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.BactchNormalization instead.') layer = BatchNormalization( axis=axis, momentum=momentum, diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py index b3535c4410..78dd617bec 100644 --- a/tensorflow/python/layers/pooling.py +++ b/tensorflow/python/layers/pooling.py @@ -148,15 +148,7 @@ def average_pooling1d(inputs, pool_size, strides, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.AveragePooling1D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.AveragePooling1D instead.') layer = AveragePooling1D(pool_size=pool_size, strides=strides, padding=padding, @@ -221,15 +213,7 @@ def max_pooling1d(inputs, pool_size, strides, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.MaxPooling1D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.MaxPooling1D instead.') layer = MaxPooling1D(pool_size=pool_size, strides=strides, padding=padding, @@ -370,15 +354,7 @@ def average_pooling2d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.AveragePooling2D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.AveragePooling2D instead.') layer = AveragePooling2D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name) @@ -446,15 +422,7 @@ def max_pooling2d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.MaxPooling2D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.MaxPooling2D instead.') layer = MaxPooling2D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name) @@ -608,15 +576,7 @@ def average_pooling3d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.AveragePooling3D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.AveragePooling3D instead.') layer = AveragePooling3D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name) @@ -688,15 +648,7 @@ def max_pooling3d(inputs, Raises: ValueError: if eager execution is enabled. - - @compatibility(eager) - Not compatible with eager execution. Use `tf.layers.MaxPooling3D` instead. - @end_compatibility """ - if context.in_eager_mode(): - raise ValueError( - 'Functional layers are currently not compatible with eager execution.' - 'Use tf.layers.MaxPooling3D instead.') layer = MaxPooling3D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, name=name) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 08be8574f3..197e5abcc9 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -208,6 +208,7 @@ class _VariableStore(object): self._vars = {} # A dictionary of the stored TensorFlow variables. self._partitioned_vars = {} # A dict of the stored PartitionedVariables. self.variable_scopes_count = {} # Count re-used variable scopes. + self._store_eager_variables = False def open_variable_scope(self, scope_name): if scope_name in self.variable_scopes_count: @@ -309,13 +310,21 @@ class _VariableStore(object): ValueError: when creating a new variable and shape is not declared, when reusing a variable and specifying a conflicting shape, or when violating reuse during variable creation. + RuntimeError: when eager execution is enabled and not called from an + EagerVariableStore. """ if custom_getter is not None and not callable(custom_getter): raise ValueError( "Passed a custom_getter which is not callable: %s" % custom_getter) if context.in_eager_mode(): - reuse = False + if not self._store_eager_variables and reuse: + raise RuntimeError( + "When eager execution is enabled variable reuse is only supported" + " when an EagerVariableStore is active. See the documentation on" + " EagerVariableStore for example usage.") + if self._store_eager_variables: + reuse = AUTO_REUSE use_resource = True # If a *_ref type is passed in an error would be triggered further down the @@ -795,7 +804,7 @@ class _VariableStore(object): dtype=variable_dtype, validate_shape=validate_shape, constraint=constraint) - if context.in_graph_mode(): + if context.in_graph_mode() or self._store_eager_variables: # In eager mode we do not want to keep default references to Variable # objects as this will prevent their memory from being released. self._vars[name] = v @@ -1177,6 +1186,48 @@ def _get_default_variable_store(): return store +@tf_contextlib.contextmanager +def with_variable_store(store): + store_collection = ops.get_collection_ref(_VARSTORE_KEY) + old = list(store_collection) + store_collection[:] = [store] + try: + yield + finally: + store_collection[:] = old + + +class EagerVariableStore(object): + """Wrapper allowing functional layers to be used with eager execution. + + When eager execution is enabled Variables get deleted when they go out of + scope, and are not stored in global collections by default. A lot of code + (mostly the functional layers in tf.layers) assumes that variables are kept in + a global list. + + EagerVariableStore can be used in conjunction with this code to make it + eager-friendly. For example, to create a dense layer, use: + + ``` + container = tfe.EagerVariableStore() + for input in dataset_iterator: + with container.as_default(): + x = tf.layers.dense(input, name="l1") + print(container.variables) # Should print the variables used in the layer. + ``` + """ + + def __init__(self): + self._store = _VariableStore() + self._store._store_eager_variables = True # pylint: disable=protected-access + + def as_default(self): + return with_variable_store(self._store) + + def variables(self): + return self._store._vars.values() # pylint: disable=protected-access + + def get_variable(name, shape=None, dtype=None, -- GitLab From 4b63f47d9f6e1876b8f7084b0c0c434a0930c070 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Mon, 30 Oct 2017 11:10:37 -0700 Subject: [PATCH 515/573] [XLA:CPU] Don't crash if someone tries to do dot(X, X) or dot(X, X^T). PiperOrigin-RevId: 173919310 --- .../xla/service/cpu/ir_emission_utils.cc | 5 +++-- .../xla/service/cpu/layout_assignment.cc | 17 +++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc index d72abede02..b99b36a55e 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc @@ -123,8 +123,9 @@ bool PotentiallyImplementedAsEigenDot(const HloInstruction& hlo) { if (hlo.opcode() == HloOpcode::kFusion && hlo.fusion_kind() == HloInstruction::FusionKind::kTransposeDot && hlo.fused_expression_root()->opcode() == HloOpcode::kDot) { - const Shape& lhs_shape = hlo.operand(0)->shape(); - const Shape& rhs_shape = hlo.operand(1)->shape(); + auto* dot = hlo.fused_expression_root(); + const Shape& lhs_shape = dot->operand(0)->shape(); + const Shape& rhs_shape = dot->operand(1)->shape(); if (ShapeUtil::HasZeroElements(lhs_shape) || ShapeUtil::HasZeroElements(rhs_shape)) { return false; diff --git a/tensorflow/compiler/xla/service/cpu/layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/layout_assignment.cc index 02e691b213..c446b6b792 100644 --- a/tensorflow/compiler/xla/service/cpu/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/cpu/layout_assignment.cc @@ -108,21 +108,26 @@ Status CpuLayoutAssignment::AddBackendConstraints( constraints->SetOperandLayout(col_major_shape(rhs_shape), dot, 1)); } else if (PotentiallyImplementedAsEigenDot(*instruction)) { const HloInstruction* dot = instruction; - const HloInstruction* lhs_instruction = dot->operand(0); - const HloInstruction* rhs_instruction = dot->operand(1); - // In order to implement `dot` with Eigen dot, the layouts of the lhs, // rhs, and output need to be row-major. // // These constraints are not hard constraints. Ideally, we should decide // which layouts to choose according to some cost model. Shape output_shape(row_major_shape(dot->shape())); + + const HloInstruction* lhs_instruction = dot->operand(0); Shape lhs_shape(row_major_shape(lhs_instruction->shape())); - Shape rhs_shape(row_major_shape(rhs_instruction->shape())); + TF_RETURN_IF_ERROR(constraints->SetOperandLayout(lhs_shape, dot, 0)); + + // dot is a kDot or a kTransposeDot fusion node. In the latter case, if + // it represents X @ X, it may have just one operand. + if (dot->operand_count() > 1) { + const HloInstruction* rhs_instruction = dot->operand(1); + Shape rhs_shape(row_major_shape(rhs_instruction->shape())); + TF_RETURN_IF_ERROR(constraints->SetOperandLayout(rhs_shape, dot, 1)); + } // Set layouts of the instructions' shapes. - TF_RETURN_IF_ERROR(constraints->SetOperandLayout(lhs_shape, dot, 0)); - TF_RETURN_IF_ERROR(constraints->SetOperandLayout(rhs_shape, dot, 1)); TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(output_shape, dot)); } else { for (int64 operand_no = 0; operand_no < instruction->operand_count(); -- GitLab From 629e6d0c103f96061d42094e32f509f76436ba35 Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Mon, 30 Oct 2017 11:23:36 -0700 Subject: [PATCH 516/573] Bugfix: Make `tf.contrib.distributions.Independent` tests not flaky. PiperOrigin-RevId: 173921378 --- .../distributions/python/kernel_tests/independent_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py index 8e23a3ab8f..06318ca09d 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py @@ -56,7 +56,7 @@ class ProductDistributionTest(test.TestCase): distribution=normal_lib.Normal(loc=loc, scale=scale), reinterpreted_batch_ndims=1) - x = ind.sample([4, 5]) + x = ind.sample([4, 5], seed=42) log_prob_x = ind.log_prob(x) x_, actual_log_prob_x = sess.run([x, log_prob_x]) @@ -79,7 +79,7 @@ class ProductDistributionTest(test.TestCase): scale_identity_multiplier=scale), reinterpreted_batch_ndims=1) - x = ind.sample([4, 5]) + x = ind.sample([4, 5], seed=42) log_prob_x = ind.log_prob(x) x_, actual_log_prob_x = sess.run([x, log_prob_x]) @@ -141,7 +141,7 @@ class ProductDistributionTest(test.TestCase): dtypes.float32, shape=logits.shape if static_shape else None) ind = independent_lib.Independent( distribution=bernoulli_lib.Bernoulli(logits=logits_ph)) - x = ind.sample(sample_shape) + x = ind.sample(sample_shape, seed=42) log_prob_x = ind.log_prob(x) [ x_, -- GitLab From 1b6b7e208f22b2f15768464e266f0fc4c235b4de Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 12:27:53 -0700 Subject: [PATCH 517/573] Add registration for op AddV2, which is identical to Add, except that it does does not implement string concatenation. This allows us to mark AddV2 is_commutative and is_aggregate, which will allow optimizers more freedom. PiperOrigin-RevId: 173931848 --- tensorflow/core/kernels/cwise_op_add_1.cc | 22 +++++++++++++++++++++- tensorflow/core/kernels/cwise_op_add_2.cc | 6 ++++++ tensorflow/core/ops/math_ops.cc | 20 +++++++++++++++++++- tensorflow/python/ops/hidden_ops.txt | 1 + 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/cwise_op_add_1.cc b/tensorflow/core/kernels/cwise_op_add_1.cc index c0fe81ef55..608a6dce3d 100644 --- a/tensorflow/core/kernels/cwise_op_add_1.cc +++ b/tensorflow/core/kernels/cwise_op_add_1.cc @@ -18,9 +18,12 @@ limitations under the License. namespace tensorflow { REGISTER5(BinaryOp, CPU, "Add", functor::add, float, Eigen::half, double, int32, int64); +REGISTER5(BinaryOp, CPU, "AddV2", functor::add, float, Eigen::half, double, + int32, int64); #if GOOGLE_CUDA REGISTER3(BinaryOp, GPU, "Add", functor::add, float, Eigen::half, double); +REGISTER3(BinaryOp, GPU, "AddV2", functor::add, float, Eigen::half, double); // A special GPU kernel for int32. // TODO(b/25387198): Also enable int32 in device memory. This kernel @@ -32,11 +35,21 @@ REGISTER_KERNEL_BUILDER(Name("Add") .HostMemory("z") .TypeConstraint("T"), BinaryOp>); +REGISTER_KERNEL_BUILDER(Name("AddV2") + .Device(DEVICE_GPU) + .HostMemory("x") + .HostMemory("y") + .HostMemory("z") + .TypeConstraint("T"), + BinaryOp>); #endif #if TENSORFLOW_USE_SYCL -#define REGISTER_KERNEL(type) REGISTER(BinaryOp, SYCL, "Add", functor::add, type); +#define REGISTER_KERNEL(type) \ + REGISTER(BinaryOp, SYCL, "Add", functor::add, type); \ + REEGISTER(BinaryOp, SYCL, "AddV2", functor::add, type); + TF_CALL_SYCL_NUMBER_TYPES(REGISTER_KERNEL); REGISTER_KERNEL_BUILDER(Name("Add") @@ -46,5 +59,12 @@ REGISTER_KERNEL_BUILDER(Name("Add") .HostMemory("z") .TypeConstraint("T"), BinaryOp>); +REGISTER_KERNEL_BUILDER(Name("AddV2") + .Device(DEVICE_SYCL) + .HostMemory("x") + .HostMemory("y") + .HostMemory("z") + .TypeConstraint("T"), + BinaryOp>); #endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_add_2.cc b/tensorflow/core/kernels/cwise_op_add_2.cc index 5dea00e95c..ac21ca06c9 100644 --- a/tensorflow/core/kernels/cwise_op_add_2.cc +++ b/tensorflow/core/kernels/cwise_op_add_2.cc @@ -24,9 +24,15 @@ namespace tensorflow { REGISTER6(BinaryOp, CPU, "Add", functor::add, int8, int16, complex64, uint8, complex128, string); +// Notice: String is excluded to allow marking AddV2 is_commutative and +// is_aggregate. +REGISTER5(BinaryOp, CPU, "AddV2", functor::add, int8, int16, complex64, uint8, + complex128); #if GOOGLE_CUDA REGISTER4(BinaryOp, GPU, "Add", functor::add, uint8, int64, complex64, complex128); +REGISTER4(BinaryOp, GPU, "AddV2", functor::add, uint8, int64, complex64, + complex128); #endif // GOOGLE_CUDA #endif // !defined(__ANDROID_TYPES_SLIM__) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 130e3ed781..045b0795ed 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -514,7 +514,6 @@ rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.] Input("x: T").Input("y: T").Output("z: T").Attr( \ "T: {half, float, double, int32, int64, complex64, complex128}") -// TODO(mrry): Restore `SetIsCommutative()` for non-string types. REGISTER_OP("Add") .Input("x: T") .Input("y: T") @@ -530,6 +529,25 @@ Returns x + y element-wise. [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); +// TODO(rmlarsen): Add a Python wrapper that swiches non-string instances to +// use AddV2 (b/68646025). +REGISTER_OP("AddV2") + .Input("x: T") + .Input("y: T") + .Output("z: T") + .Attr( + "T: {half, float, double, uint8, int8, int16, int32, int64, complex64, " + "complex128}") + .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) + .SetIsAggregate() + .SetIsCommutative() + .Doc(R"doc( +Returns x + y element-wise. + +*NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting +[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +)doc"); + REGISTER_OP("_MklAdd") .Input("x: T") .Input("y: T") diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt index 86bc038e86..732ab8f15a 100644 --- a/tensorflow/python/ops/hidden_ops.txt +++ b/tensorflow/python/ops/hidden_ops.txt @@ -244,6 +244,7 @@ TensorSummaryV2 Abs AccumulateNV2 AddN +AddV2 All Any BatchMatMul -- GitLab From b9337de5b354c7869c01f4d0cc1eb40209b6290c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 12:29:16 -0700 Subject: [PATCH 518/573] K-FAC: Multi-tower support for ConvKFCBasicFB PiperOrigin-RevId: 173932013 --- .../python/kernel_tests/fisher_blocks_test.py | 34 ++++---- .../contrib/kfac/python/ops/fisher_blocks.py | 81 +++++++++++++++---- .../contrib/kfac/python/ops/fisher_factors.py | 37 ++++++++- .../kfac/python/ops/layer_collection.py | 6 +- 4 files changed, 121 insertions(+), 37 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py index 85ac08a1eb..dbf40fccc8 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py @@ -652,10 +652,10 @@ class ConvKFCBasicFBTest(test.TestCase): params = array_ops.constant(params) inputs = random_ops.random_normal((2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - [1, 1, 1], 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, [1, 1, 1], 'SAME') + block.register_additional_minibatch(inputs, outputs) - self.assertAllEqual(outputs, block.tensors_to_compute_grads()) + self.assertAllEqual([outputs], block.tensors_to_compute_grads()) def testConvKFCBasicFBInitParamsParamsTuple(self): self._testConvKFCBasicFBInitParams([np.array([1., 2.]), np.array(3.)]) @@ -669,10 +669,11 @@ class ConvKFCBasicFBTest(test.TestCase): params = random_ops.random_normal((2, 2, 2, 2)) inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -694,11 +695,12 @@ class ConvKFCBasicFBTest(test.TestCase): params = random_ops.random_normal((2, 2, 2, 2)) inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) self.assertFalse(block._has_bias) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -716,11 +718,12 @@ class ConvKFCBasicFBTest(test.TestCase): params = [random_ops.random_normal((2, 2, 2, 2))] inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) self.assertTrue(block._has_bias) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -738,11 +741,12 @@ class ConvKFCBasicFBTest(test.TestCase): params = array_ops.zeros((2, 2, 2, 2)) inputs = array_ops.zeros((2, 2, 2, 2)) outputs = array_ops.zeros((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 damping = 0. # This test is only valid without damping. - block.instantiate_factors((grads,), damping) + block.instantiate_factors(([grads],), damping) sess.run(state_ops.assign(block._input_factor._cov, _make_psd(8))) sess.run(state_ops.assign(block._output_factor._cov, _make_psd(2))) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 7ef755c35e..efffaaef8d 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -454,6 +454,14 @@ class KroneckerProductFB(FisherBlock): @property def _renorm_coeff(self): + """Kronecker factor multiplier coefficient. + + If this FisherBlock is represented as 'FB = c * kron(left, right)', then + this is 'c'. + + Returns: + 0-D Tensor. + """ return 1.0 def multiply_inverse(self, vector): @@ -560,17 +568,34 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB): @property def num_registered_minibatches(self): - return 1 # Multiple minibatches not supported. + return len(self._inputs) class ConvKFCBasicFB(KroneckerProductFB): """FisherBlock for 2D convolutional layers using the basic KFC approx. - See https://arxiv.org/abs/1602.01407 for details. + Estimates the Fisher Information matrix's blog for a convolutional + layer. + + Consider a convoluational layer in this model with (unshared) filter matrix + 'w'. For a minibatch that produces inputs 'a' and output preactivations 's', + this FisherBlock estimates, + + F(w) = #locations * kronecker(E[flat(a) flat(a)^T], + E[flat(ds) flat(ds)^T]) + + where + + ds = (d / ds) log p(y | x, w) + #locations = number of (x, y) locations where 'w' is applied. + + where the expectation is taken over all examples and locations and flat() + concatenates an array's leading dimensions. + + See equation 23 in https://arxiv.org/abs/1602.01407 for details. """ - def __init__(self, layer_collection, params, inputs, outputs, strides, - padding): + def __init__(self, layer_collection, params, strides, padding): """Creates a ConvKFCBasicFB block. Args: @@ -580,38 +605,43 @@ class ConvKFCBasicFB(KroneckerProductFB): kernel alone, a Tensor of shape [kernel_height, kernel_width, in_channels, out_channels]. If kernel and bias, a tuple of 2 elements containing the previous and a Tensor of shape [out_channels]. - inputs: A Tensor of shape [batch_size, height, width, in_channels]. - Input activations to this layer. - outputs: A Tensor of shape [batch_size, height, width, out_channels]. - Output pre-activations from this layer. strides: The stride size in this layer (1-D Tensor of length 4). padding: The padding in this layer (1-D of Tensor length 4). """ - self._inputs = inputs - self._outputs = outputs - self._strides = strides + self._inputs = [] + self._outputs = [] + self._strides = tuple(strides) if isinstance(strides, list) else strides self._padding = padding self._has_bias = isinstance(params, (tuple, list)) fltr = params[0] if self._has_bias else params self._filter_shape = tuple(fltr.shape.as_list()) - input_shape = tuple(inputs.shape.as_list()) - self._num_locations = ( - input_shape[1] * input_shape[2] // (strides[1] * strides[2])) - super(ConvKFCBasicFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): + # TODO(b/68033310): Validate which of, + # (1) summing on a single device (as below), or + # (2) on each device in isolation and aggregating + # is faster. + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + + # Infer number of locations upon which convolution is applied. + self._num_locations = _num_conv_locations(inputs.shape.as_list(), + self._strides) + self._input_factor = self._layer_collection.make_or_get_factor( fisher_factors.ConvInputKroneckerFactor, - (self._inputs, self._filter_shape, self._strides, self._padding, + (inputs, self._filter_shape, self._strides, self._padding, self._has_bias)) self._output_factor = self._layer_collection.make_or_get_factor( fisher_factors.ConvOutputKroneckerFactor, (grads_list,)) if NORMALIZE_DAMPING_POWER: damping /= self._num_locations**NORMALIZE_DAMPING_POWER + self._damping = damping + self._register_damped_input_and_output_inverses(damping) @property @@ -621,9 +651,21 @@ class ConvKFCBasicFB(KroneckerProductFB): def tensors_to_compute_grads(self): return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, height, width, input_size]. Inputs to + the convolution. + outputs: Tensor of shape [batch_size, height, width, output_size]. Layer + preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + @property def num_registered_minibatches(self): - return 1 # Multiple minibatches not supported. + return len(self._inputs) def _concat_along_batch_dim(tensor_list): @@ -651,3 +693,8 @@ def _concat_along_batch_dim(tensor_list): else: # [tensor1, tensor2] --> tensor return array_ops.concat(tensor_list, axis=0) + + +def _num_conv_locations(input_shape, strides): + """Returns the number of locations a Conv kernel is applied to.""" + return input_shape[1] * input_shape[2] // (strides[1] * strides[2]) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index b8b524406c..4e36813369 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -609,9 +609,28 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor): class ConvInputKroneckerFactor(InverseProvidingFactor): - """Kronecker factor for the input side of a convolutional layer.""" + r"""Kronecker factor for the input side of a convolutional layer. + + Estimates E[ a a^T ] where a is the inputs to a convolutional layer given + example x. Expectation is taken over all examples and locations. + + Equivalent to \Omega in https://arxiv.org/abs/1602.01407 for details. See + Section 3.1 Estimating the factors. + """ def __init__(self, inputs, filter_shape, strides, padding, has_bias=False): + """Initializes ConvInputKroneckerFactor. + + Args: + inputs: Tensor of shape [batch_size, height, width, in_channels]. Inputs + to layer. + filter_shape: 1-D Tensor of length 4. Contains [kernel_height, + kernel_width, in_channels, out_channels]. + strides: 1-D Tensor of length 4. Contains [batch_stride, height_stride, + width_stride, in_channel_stride]. + padding: str. Padding method for layer. "SAME" or "VALID". + has_bias: bool. If True, append 1 to in_channel. + """ self._filter_shape = filter_shape self._strides = strides self._padding = padding @@ -659,9 +678,23 @@ class ConvInputKroneckerFactor(InverseProvidingFactor): class ConvOutputKroneckerFactor(InverseProvidingFactor): - """Kronecker factor for the output side of a convolutional layer.""" + r"""Kronecker factor for the output side of a convolutional layer. + + Estimates E[ ds ds^T ] where s is the preactivations of a convolutional layer + given example x and ds = (d / d s) log(p(y|x, w)). Expectation is taken over + all examples and locations. + + Equivalent to \Gamma in https://arxiv.org/abs/1602.01407 for details. See + Section 3.1 Estimating the factors. + """ def __init__(self, outputs_grads): + """Initializes ConvOutputKroneckerFactor. + + Args: + outputs_grads: list of Tensors. Each Tensor is of shape + [batch_size, height, width, out_channels]. + """ self._out_channels = outputs_grads[0].shape.as_list()[3] self._outputs_grads = outputs_grads super(ConvOutputKroneckerFactor, self).__init__() diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 2b9958a46a..77ddd19e59 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -315,9 +315,9 @@ class LayerCollection(object): approx=APPROX_KRONECKER_NAME): if approx == APPROX_KRONECKER_NAME: - self.register_block(params, - fb.ConvKFCBasicFB(self, params, inputs, outputs, - strides, padding)) + block = fb.ConvKFCBasicFB(self, params, strides, padding) + block.register_additional_minibatch(inputs, outputs) + self.register_block(params, block) elif approx == APPROX_DIAGONAL_NAME: block = fb.ConvDiagonalFB(self, params, strides, padding) block.register_additional_minibatch(inputs, outputs) -- GitLab From 8cc7b47a4f3b716a5df10a7659ec3927f07115bb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 12:33:36 -0700 Subject: [PATCH 519/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173932574 --- .../core/ops/compat/ops_history.v1.pbtxt | 35 ++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 37 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 4d00694707..35df6e89fa 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -557,6 +557,41 @@ op { } is_stateful: true } +op { + name: "AddV2" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + is_aggregate: true + is_commutative: true +} op { name: "AdjustContrast" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index f41cb212be..9f255f13c4 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -446,6 +446,43 @@ op { description: "A `SparseTensor` is represented by three tensors: `sparse_indices`,\n`sparse_values`, and `sparse_shape`.\n\nThis operator takes the given `SparseTensor` and adds it to a container\nobject (a `SparseTensorsMap`). A unique key within this container is generated\nin the form of an `int64`, and this is the value that is returned.\n\nThe `SparseTensor` can then be read out as part of a minibatch by passing\nthe key as a vector element to `TakeManySparseFromTensorsMap`. To ensure\nthe correct `SparseTensorsMap` is accessed, ensure that the same\n`container` and `shared_name` are passed to that Op. If no `shared_name`\nis provided here, instead use the *name* of the Operation created by calling\n`AddSparseToTensorsMap` as the `shared_name` passed to\n`TakeManySparseFromTensorsMap`. Ensure the Operations are colocated." is_stateful: true } +op { + name: "AddV2" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + summary: "Returns x + y element-wise." + description: "*NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" + is_aggregate: true + is_commutative: true +} op { name: "AdjustContrast" input_arg { -- GitLab From 32f3c3a4313ca50205caf022a10f6c1e9c5eb824 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 12:39:06 -0700 Subject: [PATCH 520/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173933228 --- tensorflow/go/op/wrappers.go | 130 ++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 56 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index ebe4a51116..2f8a06a632 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -7940,6 +7940,62 @@ func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) { return op.Output(0) } +// Writes contents to the file at input filename. Creates file and recursively +// +// creates directory if not existing. +// +// Arguments: +// filename: scalar. The name of the file to which we write the contents. +// contents: scalar. The content to be written to the output file. +// +// Returns the created operation. +func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "WriteFile", + Input: []tf.Input{ + filename, contents, + }, + } + return scope.AddOperation(opspec) +} + +// Computes the Cholesky decomposition of one or more square matrices. +// +// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices. +// +// The input has to be symmetric and positive definite. Only the lower-triangular +// part of the input will be used for this operation. The upper-triangular part +// will not be read. +// +// The output is a tensor of the same shape as the input +// containing the Cholesky decompositions for all input submatrices `[..., :, :]`. +// +// **Note**: The gradient computation on GPU is faster for large matrices but +// not for large batch dimensions when the submatrices are small. In this +// case it might be faster to use the CPU. +// +// Arguments: +// input: Shape is `[..., M, M]`. +// +// Returns Shape is `[..., M, M]`. +func Cholesky(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Cholesky", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad. type FusedBatchNormGradAttr func(optionalAttr) @@ -12667,62 +12723,6 @@ func AvgPool(scope *Scope, value tf.Output, ksize []int64, strides []int64, padd return op.Output(0) } -// Writes contents to the file at input filename. Creates file and recursively -// -// creates directory if not existing. -// -// Arguments: -// filename: scalar. The name of the file to which we write the contents. -// contents: scalar. The content to be written to the output file. -// -// Returns the created operation. -func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "WriteFile", - Input: []tf.Input{ - filename, contents, - }, - } - return scope.AddOperation(opspec) -} - -// Computes the Cholesky decomposition of one or more square matrices. -// -// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices. -// -// The input has to be symmetric and positive definite. Only the lower-triangular -// part of the input will be used for this operation. The upper-triangular part -// will not be read. -// -// The output is a tensor of the same shape as the input -// containing the Cholesky decompositions for all input submatrices `[..., :, :]`. -// -// **Note**: The gradient computation on GPU is faster for large matrices but -// not for large batch dimensions when the submatrices are small. In this -// case it might be faster to use the CPU. -// -// Arguments: -// input: Shape is `[..., M, M]`. -// -// Returns Shape is `[..., M, M]`. -func Cholesky(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Cholesky", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Slice a `SparseTensor` based on the `start` and `size`. // // For example, if the input is @@ -24395,6 +24395,24 @@ func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { return op.Output(0) } +// Returns x + y element-wise. +// +// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func AddV2(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "AddV2", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Saves the input tensors to disk. // // The size of `tensor_names` must match the number of tensors in `data`. `data[i]` -- GitLab From 35cc8bb0a292091a20d821644bf9732f6a98d2f0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 13:34:59 -0700 Subject: [PATCH 521/573] K-FAC: Multiple minibatches support for LayerCollection.register_conv2d() PiperOrigin-RevId: 173941279 --- .../kernel_tests/layer_collection_test.py | 124 ++++++++++-------- .../kfac/python/ops/layer_collection.py | 67 ++++++++-- 2 files changed, 125 insertions(+), 66 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 4f27ceced9..db7ab63c7d 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -329,72 +329,82 @@ class LayerCollectionTest(test.TestCase): single_loss = sess.run(lc.total_loss()) self.assertAlmostEqual(7.6983433, single_loss) + def ensureLayerReuseWorks(self, register_fn): + """Ensure the 'reuse' keyword argument function as intended. + + Args: + register_fn: function for registering a layer. Arguments are + layer_collection, reuse, and approx. + """ + # Fails on second if reuse=False. + lc = layer_collection.LayerCollection() + register_fn(lc) + with self.assertRaises(ValueError): + register_fn(lc, reuse=False) + + # Succeeds on second if reuse=True. + lc = layer_collection.LayerCollection() + register_fn(lc) + register_fn(lc, reuse=True) + + # Fails on second if reuse=VARIABLE_SCOPE and no variable reuse. + lc = layer_collection.LayerCollection() + register_fn(lc) + with self.assertRaises(ValueError): + register_fn(lc, reuse=layer_collection.VARIABLE_SCOPE) + + # Succeeds on second if reuse=VARIABLE_SCOPE and variable reuse. + lc = layer_collection.LayerCollection() + register_fn(lc) + with variable_scope.variable_scope( + variable_scope.get_variable_scope(), reuse=True): + register_fn(lc, reuse=layer_collection.VARIABLE_SCOPE) + + # Fails if block type changes. + lc = layer_collection.LayerCollection() + register_fn(lc, approx=layer_collection.APPROX_KRONECKER_NAME) + with self.assertRaises(ValueError): + register_fn(lc, approx=layer_collection.APPROX_DIAGONAL_NAME, reuse=True) + + # Fails if reuse requested but no FisherBlock exists. + lc = layer_collection.LayerCollection() + with self.assertRaises(KeyError): + register_fn(lc, reuse=True) + def testRegisterFullyConnectedReuse(self): - """Ensure the 'reuse' keyword argument function as intended.""" + """Ensure the 'reuse' works with register_fully_connected.""" with ops.Graph().as_default(): - inputs = [ - array_ops.ones([2, 10]), # - array_ops.zeros([5, 10]) - ] - outputs = [ - array_ops.zeros([2, 5]), # - array_ops.ones([5, 5]) - ] + inputs = array_ops.ones([2, 10]) + outputs = array_ops.zeros([2, 5]) params = ( variable_scope.get_variable('w', [10, 5]), # variable_scope.get_variable('b', [5])) - # Fails on second if reuse=False. - lc = layer_collection.LayerCollection() - lc.register_fully_connected(params, inputs[0], outputs[0]) - with self.assertRaises(ValueError): - lc.register_fully_connected(params, inputs[1], outputs[1], reuse=False) - - # Succeeds on second if reuse=True. - lc = layer_collection.LayerCollection() - lc.register_fully_connected(params, inputs[0], outputs[0]) - lc.register_fully_connected(params, inputs[1], outputs[1], reuse=True) - - # Fails on second if reuse=VARIABLE_SCOPE and no variable reuse. - lc = layer_collection.LayerCollection() - lc.register_fully_connected(params, inputs[0], outputs[0]) - with self.assertRaises(ValueError): - lc.register_fully_connected( - params, - inputs[1], - outputs[1], - reuse=layer_collection.VARIABLE_SCOPE) - - # Succeeds on second if reuse=VARIABLE_SCOPE and variable reuse. - lc = layer_collection.LayerCollection() - lc.register_fully_connected(params, inputs[0], outputs[0]) - with variable_scope.variable_scope( - variable_scope.get_variable_scope(), reuse=True): + def register_fn(lc, **kwargs): lc.register_fully_connected( - params, - inputs[1], - outputs[1], - reuse=layer_collection.VARIABLE_SCOPE) + params=params, inputs=inputs, outputs=outputs, **kwargs) - # Fails if block type changes. - lc = layer_collection.LayerCollection() - lc.register_fully_connected( - params, - inputs[0], - outputs[0], - approx=layer_collection.APPROX_KRONECKER_NAME) - with self.assertRaises(ValueError): - lc.register_fully_connected( - params, - inputs[1], - outputs[1], - approx=layer_collection.APPROX_DIAGONAL_NAME, - reuse=True) + self.ensureLayerReuseWorks(register_fn) - # Fails if reuse requested but no FisherBlock exists. - lc = layer_collection.LayerCollection() - with self.assertRaises(KeyError): - lc.register_fully_connected(params, inputs[0], outputs[0], reuse=True) + def testRegisterConv2dReuse(self): + """Ensure the 'reuse' works with register_conv2d.""" + with ops.Graph().as_default(): + inputs = array_ops.ones([2, 5, 5, 10]) + outputs = array_ops.zeros([2, 5, 5, 3]) + params = ( + variable_scope.get_variable('w', [1, 1, 10, 3]), # + variable_scope.get_variable('b', [3])) + + def register_fn(lc, **kwargs): + lc.register_conv2d( + params=params, + strides=[1, 1, 1, 1], + padding='SAME', + inputs=inputs, + outputs=outputs, + **kwargs) + + self.ensureLayerReuseWorks(register_fn) def testMakeOrGetFactor(self): with ops.Graph().as_default(): diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 77ddd19e59..1806f5d865 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -311,18 +311,67 @@ class LayerCollection(object): block.register_additional_minibatch(inputs, outputs) - def register_conv2d(self, params, strides, padding, inputs, outputs, - approx=APPROX_KRONECKER_NAME): + def register_conv2d(self, + params, + strides, + padding, + inputs, + outputs, + approx=APPROX_KRONECKER_NAME, + reuse=VARIABLE_SCOPE): + """Registers a convolutional layer. - if approx == APPROX_KRONECKER_NAME: - block = fb.ConvKFCBasicFB(self, params, strides, padding) - block.register_additional_minibatch(inputs, outputs) - self.register_block(params, block) - elif approx == APPROX_DIAGONAL_NAME: - block = fb.ConvDiagonalFB(self, params, strides, padding) - block.register_additional_minibatch(inputs, outputs) + Args: + params: Tensor or 2-tuple of Tensors corresponding to weight and bias of + this layer. Weight matrix should have shape [kernel_height, + kernel_width, in_channels, out_channels]. Bias should have shape + [out_channels]. + strides: 1-D Tensor of length 4. Strides for convolution kernel. + padding: string. see tf.nn.conv2d for valid values. + inputs: Tensor of shape [batch_size, height, width, in_channels]. Inputs + to layer. + outputs: Tensor of shape [batch_size, height, width, out_channels]. + Preactivations produced by layer. + approx: str. One of APPROX_KRONECKER_NAME or APPROX_DIAGONAL_NAME. + reuse: bool or str. If True, reuse an existing FisherBlock. If False, + create a new FisherBlock. If VARIABLE_SCOPE, use + tf.get_variable_scope().reuse. + + Raises: + ValueError: For improper value to 'approx'. + KeyError: If reuse == True but no FisherBlock found for 'params'. + ValueError: If reuse == True and FisherBlock found but of the wrong type. + """ + approx_to_block_types = { + APPROX_KRONECKER_NAME: fb.ConvKFCBasicFB, + APPROX_DIAGONAL_NAME: fb.ConvDiagonalFB, + } + + if approx not in approx_to_block_types: + raise ValueError("Bad value {} for approx.".format(approx)) + + block_type = approx_to_block_types[approx] + + if reuse == VARIABLE_SCOPE: + reuse = variable_scope.get_variable_scope().reuse + + if reuse: + block = self.fisher_blocks.get(params, None) + if block is None: + raise KeyError( + "Reuse requested but no FisherBlock found for params {}.".format( + params)) + if not isinstance(block, block_type): + raise ValueError( + "Requested block of type {} but block of type {} already exists " + "for params {}.".format(block_type, type(block), params)) + + else: + block = block_type(self, params, strides, padding) self.register_block(params, block) + block.register_additional_minibatch(inputs, outputs) + def register_generic(self, params, batch_size, approx=APPROX_DIAGONAL_NAME): params = params if isinstance(params, (tuple, list)) else (params,) self._generic_registrations |= set(params) -- GitLab From 07584221f801ae1f65e75f239ff14b1d6c0596cc Mon Sep 17 00:00:00 2001 From: Anna R Date: Mon, 30 Oct 2017 13:39:50 -0700 Subject: [PATCH 522/573] Set visibility to HIDDEN for hidden Python ops in ApiDef. PiperOrigin-RevId: 173942001 --- .../core/api_def/python_api/api_def_A.pbtxt | 56 +++++ .../core/api_def/python_api/api_def_B.pbtxt | 124 ++++++++++ .../core/api_def/python_api/api_def_C.pbtxt | 44 ++++ .../core/api_def/python_api/api_def_D.pbtxt | 20 ++ .../core/api_def/python_api/api_def_E.pbtxt | 16 ++ .../core/api_def/python_api/api_def_F.pbtxt | 52 +++++ .../core/api_def/python_api/api_def_G.pbtxt | 16 ++ .../core/api_def/python_api/api_def_H.pbtxt | 12 + .../core/api_def/python_api/api_def_I.pbtxt | 40 ++++ .../core/api_def/python_api/api_def_L.pbtxt | 72 ++++++ .../core/api_def/python_api/api_def_M.pbtxt | 96 ++++++++ .../core/api_def/python_api/api_def_N.pbtxt | 16 ++ .../core/api_def/python_api/api_def_O.pbtxt | 4 + .../core/api_def/python_api/api_def_P.pbtxt | 68 ++++++ .../core/api_def/python_api/api_def_Q.pbtxt | 56 +++++ .../core/api_def/python_api/api_def_R.pbtxt | 156 +++++++++++++ .../core/api_def/python_api/api_def_S.pbtxt | 216 ++++++++++++++++++ .../core/api_def/python_api/api_def_T.pbtxt | 196 ++++++++++++++++ .../core/api_def/python_api/api_def_U.pbtxt | 8 + .../core/api_def/python_api/api_def_V.pbtxt | 8 + .../core/api_def/python_api/api_def_W.pbtxt | 8 + .../core/api_def/python_api/api_def_Z.pbtxt | 4 + tensorflow/python/BUILD | 6 + tensorflow/tools/api/tests/BUILD | 1 + .../tools/api/tests/api_compatibility_test.py | 52 ++++- 25 files changed, 1342 insertions(+), 5 deletions(-) create mode 100644 tensorflow/core/api_def/python_api/api_def_A.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_G.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_N.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_O.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_P.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_T.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_U.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_V.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_W.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_Z.pbtxt diff --git a/tensorflow/core/api_def/python_api/api_def_A.pbtxt b/tensorflow/core/api_def/python_api/api_def_A.pbtxt new file mode 100644 index 0000000000..df9b3ad0b6 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_A.pbtxt @@ -0,0 +1,56 @@ +op { + graph_op_name: "Abs" + visibility: HIDDEN +} +op { + graph_op_name: "AddManySparseToTensorsMap" + visibility: HIDDEN +} +op { + graph_op_name: "AddN" + visibility: HIDDEN +} +op { + graph_op_name: "AddSparseToTensorsMap" + visibility: HIDDEN +} +op { + graph_op_name: "AdjustContrastv2" + visibility: HIDDEN +} +op { + graph_op_name: "All" + visibility: HIDDEN +} +op { + graph_op_name: "AllCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "Any" + visibility: HIDDEN +} +op { + graph_op_name: "Assert" + visibility: HIDDEN +} +op { + graph_op_name: "AudioSummary" + visibility: HIDDEN +} +op { + graph_op_name: "AudioSummaryV2" + visibility: HIDDEN +} +op { + graph_op_name: "AvgPool" + visibility: HIDDEN +} +op { + graph_op_name: "AvgPool3DGrad" + visibility: HIDDEN +} +op { + graph_op_name: "AvgPoolGrad" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_B.pbtxt b/tensorflow/core/api_def/python_api/api_def_B.pbtxt index 9b5df58eba..49c74ccad2 100644 --- a/tensorflow/core/api_def/python_api/api_def_B.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_B.pbtxt @@ -1,3 +1,115 @@ +op { + graph_op_name: "Barrier" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierClose" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierIncompleteSize" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierInsertMany" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierReadySize" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierTakeMany" + visibility: HIDDEN +} +op { + graph_op_name: "BatchCholesky" + visibility: HIDDEN +} +op { + graph_op_name: "BatchCholeskyGrad" + visibility: HIDDEN +} +op { + graph_op_name: "BatchFFT" + visibility: HIDDEN +} +op { + graph_op_name: "BatchFFT2D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchFFT3D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchIFFT" + visibility: HIDDEN +} +op { + graph_op_name: "BatchIFFT2D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchIFFT3D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatMul" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixDeterminant" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixInverse" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixSolve" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixSolveLs" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixTriangularSolve" + visibility: HIDDEN +} +op { + graph_op_name: "BatchNormWithGlobalNormalization" + visibility: HIDDEN +} +op { + graph_op_name: "BatchNormWithGlobalNormalizationGrad" + visibility: HIDDEN +} +op { + graph_op_name: "BatchSelfAdjointEig" + visibility: HIDDEN +} +op { + graph_op_name: "BatchSelfAdjointEigV2" + visibility: HIDDEN +} +op { + graph_op_name: "BatchSvd" + visibility: HIDDEN +} +op { + graph_op_name: "BatchToSpace" + visibility: HIDDEN +} +op { + graph_op_name: "BiasAdd" + visibility: HIDDEN +} +op { + graph_op_name: "BiasAddV1" + visibility: HIDDEN +} op { graph_op_name: "BitwiseAnd" endpoint { @@ -16,3 +128,15 @@ op { name: "bitwise.bitwise_xor" } } +op { + graph_op_name: "BroadcastArgs" + visibility: HIDDEN +} +op { + graph_op_name: "BroadcastGradientArgs" + visibility: HIDDEN +} +op { + graph_op_name: "Bucketize" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_C.pbtxt b/tensorflow/core/api_def/python_api/api_def_C.pbtxt index cf8d0622be..42ed24b133 100644 --- a/tensorflow/core/api_def/python_api/api_def_C.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_C.pbtxt @@ -1,3 +1,15 @@ +op { + graph_op_name: "CTCBeamSearchDecoder" + visibility: HIDDEN +} +op { + graph_op_name: "CTCGreedyDecoder" + visibility: HIDDEN +} +op { + graph_op_name: "CTCLoss" + visibility: HIDDEN +} op { graph_op_name: "Cholesky" endpoint { @@ -7,6 +19,38 @@ op { name: "linalg.cholesky" } } +op { + graph_op_name: "Complex" + visibility: HIDDEN +} +op { + graph_op_name: "ComplexAbs" + visibility: HIDDEN +} +op { + graph_op_name: "ComputeAccidentalHits" + visibility: HIDDEN +} +op { + graph_op_name: "Concat" + visibility: HIDDEN +} +op { + graph_op_name: "ConcatOffset" + visibility: HIDDEN +} +op { + graph_op_name: "ConcatV2" + visibility: HIDDEN +} +op { + graph_op_name: "Conj" + visibility: HIDDEN +} +op { + graph_op_name: "Const" + visibility: HIDDEN +} op { graph_op_name: "CropAndResize" endpoint { diff --git a/tensorflow/core/api_def/python_api/api_def_D.pbtxt b/tensorflow/core/api_def/python_api/api_def_D.pbtxt index 12e0dbec1c..c73982aed0 100644 --- a/tensorflow/core/api_def/python_api/api_def_D.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_D.pbtxt @@ -1,3 +1,7 @@ +op { + graph_op_name: "DebugGradientIdentity" + visibility: HIDDEN +} op { graph_op_name: "DecodeAndCropJpeg" endpoint { @@ -10,6 +14,10 @@ op { name: "image.decode_bmp" } } +op { + graph_op_name: "DecodeCSV" + visibility: HIDDEN +} op { graph_op_name: "DecodeGif" endpoint { @@ -28,6 +36,10 @@ op { name: "image.decode_png" } } +op { + graph_op_name: "DeleteSessionTensor" + visibility: HIDDEN +} op { graph_op_name: "DepthwiseConv2dNative" endpoint { @@ -46,6 +58,14 @@ op { name: "nn.depthwise_conv2d_native_backprop_input" } } +op { + graph_op_name: "DeserializeManySparse" + visibility: HIDDEN +} +op { + graph_op_name: "DestroyTemporaryVariable" + visibility: HIDDEN +} op { graph_op_name: "DrawBoundingBoxes" endpoint { diff --git a/tensorflow/core/api_def/python_api/api_def_E.pbtxt b/tensorflow/core/api_def/python_api/api_def_E.pbtxt index f6871f7138..236c344167 100644 --- a/tensorflow/core/api_def/python_api/api_def_E.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_E.pbtxt @@ -1,9 +1,17 @@ +op { + graph_op_name: "EditDistance" + visibility: HIDDEN +} op { graph_op_name: "Elu" endpoint { name: "nn.elu" } } +op { + graph_op_name: "EluGrad" + visibility: HIDDEN +} op { graph_op_name: "EncodeJpeg" endpoint { @@ -16,6 +24,14 @@ op { name: "image.encode_png" } } +op { + graph_op_name: "Exit" + visibility: HIDDEN +} +op { + graph_op_name: "ExpandDims" + visibility: HIDDEN +} op { graph_op_name: "ExtractGlimpse" endpoint { diff --git a/tensorflow/core/api_def/python_api/api_def_F.pbtxt b/tensorflow/core/api_def/python_api/api_def_F.pbtxt index 844a1348a3..a29b6a3725 100644 --- a/tensorflow/core/api_def/python_api/api_def_F.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_F.pbtxt @@ -7,15 +7,67 @@ op { name: "spectral.fft" } } +op { + graph_op_name: "FIFOQueue" + visibility: HIDDEN +} +op { + graph_op_name: "FIFOQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "Fact" + visibility: HIDDEN +} +op { + graph_op_name: "FakeQueue" + visibility: HIDDEN +} +op { + graph_op_name: "FixedLengthRecordReader" + visibility: HIDDEN +} +op { + graph_op_name: "FixedLengthRecordReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "FixedUnigramCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "FloorDiv" + visibility: HIDDEN +} +op { + graph_op_name: "FloorMod" + visibility: HIDDEN +} op { graph_op_name: "FractionalAvgPool" endpoint { name: "nn.fractional_avg_pool" } } +op { + graph_op_name: "FractionalAvgPoolGrad" + visibility: HIDDEN +} op { graph_op_name: "FractionalMaxPool" endpoint { name: "nn.fractional_max_pool" } } +op { + graph_op_name: "FractionalMaxPoolGrad" + visibility: HIDDEN +} +op { + graph_op_name: "FusedBatchNorm" + visibility: HIDDEN +} +op { + graph_op_name: "FusedBatchNormV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_G.pbtxt b/tensorflow/core/api_def/python_api/api_def_G.pbtxt new file mode 100644 index 0000000000..8235d245fe --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_G.pbtxt @@ -0,0 +1,16 @@ +op { + graph_op_name: "GenerateVocabRemapping" + visibility: HIDDEN +} +op { + graph_op_name: "GetSessionHandle" + visibility: HIDDEN +} +op { + graph_op_name: "GetSessionHandleV2" + visibility: HIDDEN +} +op { + graph_op_name: "GetSessionTensor" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_H.pbtxt b/tensorflow/core/api_def/python_api/api_def_H.pbtxt index 55998189f4..9f3fe2eb08 100644 --- a/tensorflow/core/api_def/python_api/api_def_H.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_H.pbtxt @@ -4,3 +4,15 @@ op { name: "image.hsv_to_rgb" } } +op { + graph_op_name: "HashTable" + visibility: HIDDEN +} +op { + graph_op_name: "HashTableV2" + visibility: HIDDEN +} +op { + graph_op_name: "HistogramSummary" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_I.pbtxt b/tensorflow/core/api_def/python_api/api_def_I.pbtxt index 6c794fab0d..db6a54dbd4 100644 --- a/tensorflow/core/api_def/python_api/api_def_I.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_I.pbtxt @@ -7,6 +7,46 @@ op { name: "spectral.ifft" } } +op { + graph_op_name: "IdentityReader" + visibility: HIDDEN +} +op { + graph_op_name: "IdentityReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "ImageSummary" + visibility: HIDDEN +} +op { + graph_op_name: "InTopK" + visibility: HIDDEN +} +op { + graph_op_name: "InTopKV2" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTable" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTableFromTextFile" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTableFromTextFileV2" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTableV2" + visibility: HIDDEN +} +op { + graph_op_name: "InvGrad" + visibility: HIDDEN +} op { graph_op_name: "Invert" endpoint { diff --git a/tensorflow/core/api_def/python_api/api_def_L.pbtxt b/tensorflow/core/api_def/python_api/api_def_L.pbtxt index 38ba26a8e8..083fbdae6f 100644 --- a/tensorflow/core/api_def/python_api/api_def_L.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_L.pbtxt @@ -4,6 +4,10 @@ op { name: "nn.l2_loss" } } +op { + graph_op_name: "LMDBReader" + visibility: HIDDEN +} op { graph_op_name: "LRN" endpoint { @@ -13,6 +17,14 @@ op { name: "nn.lrn" } } +op { + graph_op_name: "LRNGrad" + visibility: HIDDEN +} +op { + graph_op_name: "LearnedUnigramCandidateSampler" + visibility: HIDDEN +} op { graph_op_name: "LinSpace" endpoint { @@ -22,3 +34,63 @@ op { name: "linspace" } } +op { + graph_op_name: "ListDiff" + visibility: HIDDEN +} +op { + graph_op_name: "LoadAndRemapMatrix" + visibility: HIDDEN +} +op { + graph_op_name: "LogMatrixDeterminant" + visibility: HIDDEN +} +op { + graph_op_name: "LogSoftmax" + visibility: HIDDEN +} +op { + graph_op_name: "LogUniformCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableExport" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableExportV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableFind" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableFindV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableImport" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableImportV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableInsert" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableInsertV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableSize" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableSizeV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_M.pbtxt b/tensorflow/core/api_def/python_api/api_def_M.pbtxt index 154071f6bc..c8840e0c09 100644 --- a/tensorflow/core/api_def/python_api/api_def_M.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_M.pbtxt @@ -1,3 +1,7 @@ +op { + graph_op_name: "MatMul" + visibility: HIDDEN +} op { graph_op_name: "MatrixBandPart" endpoint { @@ -61,6 +65,10 @@ op { name: "matrix_solve" } } +op { + graph_op_name: "MatrixSolveLs" + visibility: HIDDEN +} op { graph_op_name: "MatrixTriangularSolve" endpoint { @@ -70,9 +78,97 @@ op { name: "matrix_triangular_solve" } } +op { + graph_op_name: "Max" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPool" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPool3DGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPool3DGradGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGradGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGradGradWithArgmax" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGradWithArgmax" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolV2" + visibility: HIDDEN +} op { graph_op_name: "MaxPoolWithArgmax" endpoint { name: "nn.max_pool_with_argmax" } } +op { + graph_op_name: "Mean" + visibility: HIDDEN +} +op { + graph_op_name: "Merge" + visibility: HIDDEN +} +op { + graph_op_name: "MergeSummary" + visibility: HIDDEN +} +op { + graph_op_name: "Min" + visibility: HIDDEN +} +op { + graph_op_name: "MirrorPad" + visibility: HIDDEN +} +op { + graph_op_name: "MirrorPadGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Mul" + visibility: HIDDEN +} +op { + graph_op_name: "MutableDenseHashTable" + visibility: HIDDEN +} +op { + graph_op_name: "MutableDenseHashTableV2" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTable" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTableOfTensors" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTableOfTensorsV2" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTableV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_N.pbtxt b/tensorflow/core/api_def/python_api/api_def_N.pbtxt new file mode 100644 index 0000000000..60da4dcafe --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_N.pbtxt @@ -0,0 +1,16 @@ +op { + graph_op_name: "Neg" + visibility: HIDDEN +} +op { + graph_op_name: "NegTrain" + visibility: HIDDEN +} +op { + graph_op_name: "NonMaxSuppression" + visibility: HIDDEN +} +op { + graph_op_name: "NonMaxSuppressionV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_O.pbtxt b/tensorflow/core/api_def/python_api/api_def_O.pbtxt new file mode 100644 index 0000000000..3a9f0f4032 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_O.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "OneHot" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_P.pbtxt b/tensorflow/core/api_def/python_api/api_def_P.pbtxt new file mode 100644 index 0000000000..87ca53e0b9 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_P.pbtxt @@ -0,0 +1,68 @@ +op { + graph_op_name: "Pack" + visibility: HIDDEN +} +op { + graph_op_name: "Pad" + visibility: HIDDEN +} +op { + graph_op_name: "PadV2" + visibility: HIDDEN +} +op { + graph_op_name: "PaddingFIFOQueue" + visibility: HIDDEN +} +op { + graph_op_name: "PaddingFIFOQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "ParallelConcat" + visibility: HIDDEN +} +op { + graph_op_name: "ParameterizedTruncatedNormal" + visibility: HIDDEN +} +op { + graph_op_name: "ParseExample" + visibility: HIDDEN +} +op { + graph_op_name: "ParseSingleSequenceExample" + visibility: HIDDEN +} +op { + graph_op_name: "Placeholder" + visibility: HIDDEN +} +op { + graph_op_name: "Pow" + visibility: HIDDEN +} +op { + graph_op_name: "Print" + visibility: HIDDEN +} +op { + graph_op_name: "PriorityQueue" + visibility: HIDDEN +} +op { + graph_op_name: "PriorityQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "Prod" + visibility: HIDDEN +} +op { + graph_op_name: "PyFunc" + visibility: HIDDEN +} +op { + graph_op_name: "PyFuncStateless" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_Q.pbtxt b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt index cba032880f..0dfb5bb703 100644 --- a/tensorflow/core/api_def/python_api/api_def_Q.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt @@ -25,3 +25,59 @@ op { name: "nn.quantized_relu_x" } } +op { + graph_op_name: "QueueClose" + visibility: HIDDEN +} +op { + graph_op_name: "QueueCloseV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeue" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueMany" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueManyV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueUpTo" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueUpToV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueue" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueueMany" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueueManyV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueSize" + visibility: HIDDEN +} +op { + graph_op_name: "QueueSizeV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_R.pbtxt b/tensorflow/core/api_def/python_api/api_def_R.pbtxt index 9a57e72be0..0c8a8a4d42 100644 --- a/tensorflow/core/api_def/python_api/api_def_R.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_R.pbtxt @@ -4,12 +4,140 @@ op { name: "image.rgb_to_hsv" } } +op { + graph_op_name: "RandomCrop" + visibility: HIDDEN +} +op { + graph_op_name: "RandomGamma" + visibility: HIDDEN +} +op { + graph_op_name: "RandomPoisson" + visibility: HIDDEN +} +op { + graph_op_name: "RandomShuffle" + visibility: HIDDEN +} +op { + graph_op_name: "RandomShuffleQueue" + visibility: HIDDEN +} +op { + graph_op_name: "RandomShuffleQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "RandomStandardNormal" + visibility: HIDDEN +} +op { + graph_op_name: "RandomUniform" + visibility: HIDDEN +} +op { + graph_op_name: "RandomUniformInt" + visibility: HIDDEN +} +op { + graph_op_name: "Range" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumRecordsProduced" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumRecordsProducedV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumWorkUnitsCompleted" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumWorkUnitsCompletedV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderRead" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReadUpTo" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReadUpToV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReadV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReset" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderResetV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderRestoreState" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderRestoreStateV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderSerializeState" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderSerializeStateV2" + visibility: HIDDEN +} +op { + graph_op_name: "RealDiv" + visibility: HIDDEN +} +op { + graph_op_name: "ReciprocalGrad" + visibility: HIDDEN +} +op { + graph_op_name: "RefExit" + visibility: HIDDEN +} +op { + graph_op_name: "RefIdentity" + visibility: HIDDEN +} +op { + graph_op_name: "RefMerge" + visibility: HIDDEN +} op { graph_op_name: "Relu" endpoint { name: "nn.relu" } } +op { + graph_op_name: "Relu6" + visibility: HIDDEN +} +op { + graph_op_name: "Relu6Grad" + visibility: HIDDEN +} +op { + graph_op_name: "ReluGrad" + visibility: HIDDEN +} op { graph_op_name: "ResizeArea" endpoint { @@ -22,15 +150,43 @@ op { name: "image.resize_bicubic" } } +op { + graph_op_name: "ResizeBicubicGrad" + visibility: HIDDEN +} op { graph_op_name: "ResizeBilinear" endpoint { name: "image.resize_bilinear" } } +op { + graph_op_name: "ResizeBilinearGrad" + visibility: HIDDEN +} op { graph_op_name: "ResizeNearestNeighbor" endpoint { name: "image.resize_nearest_neighbor" } } +op { + graph_op_name: "ResizeNearestNeighborGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Restore" + visibility: HIDDEN +} +op { + graph_op_name: "RestoreSlice" + visibility: HIDDEN +} +op { + graph_op_name: "Reverse" + visibility: HIDDEN +} +op { + graph_op_name: "RsqrtGrad" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_S.pbtxt b/tensorflow/core/api_def/python_api/api_def_S.pbtxt index 9c7a39038e..0c34730200 100644 --- a/tensorflow/core/api_def/python_api/api_def_S.pbtxt +++ b/tensorflow/core/api_def/python_api/api_def_S.pbtxt @@ -1,3 +1,23 @@ +op { + graph_op_name: "SampleDistortedBoundingBox" + visibility: HIDDEN +} +op { + graph_op_name: "SampleDistortedBoundingBoxV2" + visibility: HIDDEN +} +op { + graph_op_name: "Save" + visibility: HIDDEN +} +op { + graph_op_name: "SaveSlices" + visibility: HIDDEN +} +op { + graph_op_name: "ScalarSummary" + visibility: HIDDEN +} op { graph_op_name: "SdcaFprint" endpoint { @@ -16,21 +36,217 @@ op { name: "train.sdca_shrink_l1" } } +op { + graph_op_name: "Select" + visibility: HIDDEN +} +op { + graph_op_name: "SelfAdjointEig" + visibility: HIDDEN +} +op { + graph_op_name: "SelfAdjointEigV2" + visibility: HIDDEN +} op { graph_op_name: "Selu" endpoint { name: "nn.selu" } } +op { + graph_op_name: "SeluGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SerializeManySparse" + visibility: HIDDEN +} +op { + graph_op_name: "SerializeSparse" + visibility: HIDDEN +} +op { + graph_op_name: "ShardedFilename" + visibility: HIDDEN +} +op { + graph_op_name: "ShardedFilespec" + visibility: HIDDEN +} +op { + graph_op_name: "Sigmoid" + visibility: HIDDEN +} +op { + graph_op_name: "SigmoidGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Skipgram" + visibility: HIDDEN +} +op { + graph_op_name: "Slice" + visibility: HIDDEN +} +op { + graph_op_name: "Softmax" + visibility: HIDDEN +} +op { + graph_op_name: "SoftmaxCrossEntropyWithLogits" + visibility: HIDDEN +} op { graph_op_name: "Softplus" endpoint { name: "nn.softplus" } } +op { + graph_op_name: "SoftplusGrad" + visibility: HIDDEN +} op { graph_op_name: "Softsign" endpoint { name: "nn.softsign" } } +op { + graph_op_name: "SoftsignGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SpaceToBatch" + visibility: HIDDEN +} +op { + graph_op_name: "SparseAdd" + visibility: HIDDEN +} +op { + graph_op_name: "SparseAddGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SparseConcat" + visibility: HIDDEN +} +op { + graph_op_name: "SparseCross" + visibility: HIDDEN +} +op { + graph_op_name: "SparseFillEmptyRows" + visibility: HIDDEN +} +op { + graph_op_name: "SparseFillEmptyRowsGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SparseMatMul" + visibility: HIDDEN +} +op { + graph_op_name: "SparseReorder" + visibility: HIDDEN +} +op { + graph_op_name: "SparseReshape" + visibility: HIDDEN +} +op { + graph_op_name: "SparseSoftmaxCrossEntropyWithLogits" + visibility: HIDDEN +} +op { + graph_op_name: "SparseSplit" + visibility: HIDDEN +} +op { + graph_op_name: "SparseTensorDenseAdd" + visibility: HIDDEN +} +op { + graph_op_name: "SparseTensorDenseMatMul" + visibility: HIDDEN +} +op { + graph_op_name: "SparseToDense" + visibility: HIDDEN +} +op { + graph_op_name: "Split" + visibility: HIDDEN +} +op { + graph_op_name: "SplitV" + visibility: HIDDEN +} +op { + graph_op_name: "SqrtGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Squeeze" + visibility: HIDDEN +} +op { + graph_op_name: "Stack" + visibility: HIDDEN +} +op { + graph_op_name: "StackClose" + visibility: HIDDEN +} +op { + graph_op_name: "StackCloseV2" + visibility: HIDDEN +} +op { + graph_op_name: "StackPop" + visibility: HIDDEN +} +op { + graph_op_name: "StackPopV2" + visibility: HIDDEN +} +op { + graph_op_name: "StackPush" + visibility: HIDDEN +} +op { + graph_op_name: "StackPushV2" + visibility: HIDDEN +} +op { + graph_op_name: "StackV2" + visibility: HIDDEN +} +op { + graph_op_name: "StringSplit" + visibility: HIDDEN +} +op { + graph_op_name: "Sub" + visibility: HIDDEN +} +op { + graph_op_name: "Sum" + visibility: HIDDEN +} +op { + graph_op_name: "Svd" + visibility: HIDDEN +} +op { + graph_op_name: "Switch" + visibility: HIDDEN +} +op { + graph_op_name: "SymbolicGradient" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_T.pbtxt b/tensorflow/core/api_def/python_api/api_def_T.pbtxt new file mode 100644 index 0000000000..8011a11243 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_T.pbtxt @@ -0,0 +1,196 @@ +op { + graph_op_name: "TFRecordReader" + visibility: HIDDEN +} +op { + graph_op_name: "TFRecordReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "TakeManySparseFromTensorsMap" + visibility: HIDDEN +} +op { + graph_op_name: "Tanh" + visibility: HIDDEN +} +op { + graph_op_name: "TanhGrad" + visibility: HIDDEN +} +op { + graph_op_name: "TemporaryVariable" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArray" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayClose" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayCloseV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayCloseV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayConcat" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayConcatV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayConcatV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGather" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGatherV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGatherV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGrad" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGradV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGradV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayPack" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayRead" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayReadV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayReadV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayScatter" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayScatterV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayScatterV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySize" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySizeV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySizeV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySplit" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySplitV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySplitV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayUnpack" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayWrite" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayWriteV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayWriteV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorSummary" + visibility: HIDDEN +} +op { + graph_op_name: "TensorSummaryV2" + visibility: HIDDEN +} +op { + graph_op_name: "TextLineReader" + visibility: HIDDEN +} +op { + graph_op_name: "TextLineReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "ThreadUnsafeUnigramCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "TileGrad" + visibility: HIDDEN +} +op { + graph_op_name: "TopK" + visibility: HIDDEN +} +op { + graph_op_name: "TopKV2" + visibility: HIDDEN +} +op { + graph_op_name: "TruncateDiv" + visibility: HIDDEN +} +op { + graph_op_name: "TruncateMod" + visibility: HIDDEN +} +op { + graph_op_name: "TruncatedNormal" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_U.pbtxt b/tensorflow/core/api_def/python_api/api_def_U.pbtxt new file mode 100644 index 0000000000..d7c261c63c --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_U.pbtxt @@ -0,0 +1,8 @@ +op { + graph_op_name: "UniformCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "Unpack" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_V.pbtxt b/tensorflow/core/api_def/python_api/api_def_V.pbtxt new file mode 100644 index 0000000000..18be21a886 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_V.pbtxt @@ -0,0 +1,8 @@ +op { + graph_op_name: "Variable" + visibility: HIDDEN +} +op { + graph_op_name: "VariableV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_W.pbtxt b/tensorflow/core/api_def/python_api/api_def_W.pbtxt new file mode 100644 index 0000000000..cd8861a98f --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_W.pbtxt @@ -0,0 +1,8 @@ +op { + graph_op_name: "WholeFileReader" + visibility: HIDDEN +} +op { + graph_op_name: "WholeFileReaderV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_Z.pbtxt b/tensorflow/core/api_def/python_api/api_def_Z.pbtxt new file mode 100644 index 0000000000..5857b7cf38 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_Z.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "ZerosLike" + visibility: HIDDEN +} diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index d435ae1375..e167af96d0 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -4035,6 +4035,12 @@ filegroup( visibility = ["//tensorflow:__subpackages__"], ) +filegroup( + name = "hidden_ops", + srcs = ["ops/hidden_ops.txt"], + visibility = ["//tensorflow:__subpackages__"], +) + cuda_py_test( name = "accumulate_n_benchmark", size = "large", diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD index a913e35101..f80dd6fe5b 100644 --- a/tensorflow/tools/api/tests/BUILD +++ b/tensorflow/tools/api/tests/BUILD @@ -20,6 +20,7 @@ py_test( ":convert_from_multiline", "//tensorflow/core:base_api_def", "//tensorflow/core:python_api_def", + "//tensorflow/python:hidden_ops", "//tensorflow/tools/api/golden:api_golden", "//tensorflow/tools/api/tests:API_UPDATE_WARNING.txt", "//tensorflow/tools/api/tests:README.txt", diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py index f350c12d41..6a27f6bc42 100644 --- a/tensorflow/tools/api/tests/api_compatibility_test.py +++ b/tensorflow/tools/api/tests/api_compatibility_test.py @@ -72,6 +72,7 @@ _ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' _CONVERT_FROM_MULTILINE_SCRIPT = 'tensorflow/tools/api/tests/convert_from_multiline' _BASE_API_DIR = 'tensorflow/core/api_def/base_api' _PYTHON_API_DIR = 'tensorflow/core/api_def/python_api' +_HIDDEN_OPS_FILE = 'tensorflow/python/ops/hidden_ops.txt' def _KeyToFilePath(key): @@ -121,6 +122,21 @@ def _IsGenModule(module_name): return module_name_split[-1].startswith('gen_') +def _GetHiddenOps(): + hidden_ops_file = file_io.FileIO(_HIDDEN_OPS_FILE, 'r') + hidden_ops = set() + for line in hidden_ops_file: + line = line.strip() + if not line: + continue + if line[0] == '#': # comment line + continue + # If line is of the form "op_name # comment", only keep the op_name. + line_split = line.split('#') + hidden_ops.add(line_split[0].strip()) + return hidden_ops + + class ApiCompatibilityTest(test.TestCase): def __init__(self, *args, **kwargs): @@ -325,16 +341,39 @@ class ApiDefTest(test.TestCase): text_format.Merge( file_io.read_file_to_string(base_api_file), api_defs) for api_def in api_defs.op: - lower_case_name = self._GenerateLowerCaseOpName(api_def.graph_op_name) - name_to_base_api_def[lower_case_name] = api_def + name_to_base_api_def[api_def.graph_op_name] = api_def return name_to_base_api_def + def _AddHiddenOpOverrides(self, name_to_base_api_def, api_def_map): + """Adds ApiDef overrides to api_def_map for hidden Python ops. + + Args: + name_to_base_api_def: Map from op name to base api_def_pb2.ApiDef. + api_def_map: Map from first op name character (in caps) to + api_def_pb2.ApiDefs for Python API overrides. + """ + hidden_ops = _GetHiddenOps() + for hidden_op in hidden_ops: + if hidden_op not in name_to_base_api_def: + logging.warning('Unexpected hidden op name: %s' % hidden_op) + continue + + base_api_def = name_to_base_api_def[hidden_op] + if base_api_def.visibility != api_def_pb2.ApiDef.HIDDEN: + api_def = api_def_pb2.ApiDef() + api_def.graph_op_name = base_api_def.graph_op_name + api_def.visibility = api_def_pb2.ApiDef.HIDDEN + api_def_map[api_def.graph_op_name[0].upper()].op.extend([api_def]) + @unittest.skipUnless( sys.version_info.major == 2 and os.uname()[0] == 'Linux', 'API compabitility test goldens are generated using python2 on Linux.') def testAPIDefCompatibility(self): # Get base ApiDef name_to_base_api_def = self._GetBaseApiMap() + snake_to_camel_graph_op_names = { + self._GenerateLowerCaseOpName(name): name + for name in name_to_base_api_def.keys()} # Extract Python API visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor() public_api_visitor = public_api.PublicAPIVisitor(visitor) @@ -357,7 +396,7 @@ class ApiDefTest(test.TestCase): # Check if object is defined in gen_* module. That is, # the object has been generated from OpDef. if hasattr(obj, '__module__') and _IsGenModule(obj.__module__): - if obj.__name__ not in name_to_base_api_def: + if obj.__name__ not in snake_to_camel_graph_op_names: # Symbol might be defined only in Python and not generated from # C++ api. continue @@ -368,12 +407,15 @@ class ApiDefTest(test.TestCase): # Generate Python ApiDef overrides. for op, endpoint_names in op_to_endpoint_name.items(): + graph_op_name = snake_to_camel_graph_op_names[op.__name__] api_def = self._CreatePythonApiDef( - name_to_base_api_def[op.__name__], endpoint_names) + name_to_base_api_def[graph_op_name], endpoint_names) if api_def: - api_defs = api_def_map[op.__name__[0].upper()] + api_defs = api_def_map[graph_op_name[0].upper()] api_defs.op.extend([api_def]) + self._AddHiddenOpOverrides(name_to_base_api_def, api_def_map) + for key in _ALPHABET: # Get new ApiDef for the given key. new_api_defs_str = '' -- GitLab From 4f6e6ea4cd4f10aa18a34603dff29d9af157b5f0 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Mon, 30 Oct 2017 13:41:48 -0700 Subject: [PATCH 523/573] Fix typo in comment; NFC PiperOrigin-RevId: 173942305 --- tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h index 01e6b4c071..f49a788922 100644 --- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h +++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h @@ -56,7 +56,7 @@ class XlaCompiledCpuFunction { const void** args, void** temps)>; // StaticData represents the state necessary to run an XLA-compiled - // function. For JIT this is backed by data in XlaCompiledCpuFunctionJit; for + // function. For JIT this is backed by data in XlaJitCompiledCpuFunction; for // AOT this is backed by data compiled into the object file. struct StaticData { // The raw function to call. -- GitLab From 682a6ed64f961d73ecdde5c3b80c6188fedcf5ee Mon Sep 17 00:00:00 2001 From: Jon Shlens Date: Mon, 30 Oct 2017 13:46:18 -0700 Subject: [PATCH 524/573] Update the documentation for sample_distorted_bounding_box PiperOrigin-RevId: 173943029 --- tensorflow/python/ops/image_ops_impl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 4aef6ca85f..2946dbe81e 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -1513,7 +1513,8 @@ def sample_distorted_bounding_box(image_size, bounding_boxes, seed=None, # Generate a single distorted bounding box. begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box( tf.shape(image), - bounding_boxes=bounding_boxes) + bounding_boxes=bounding_boxes, + min_object_covered=0.1) # Draw the bounding box in an image summary. image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), @@ -1541,7 +1542,7 @@ def sample_distorted_bounding_box(image_size, bounding_boxes, seed=None, seed. seed2: An optional `int`. Defaults to `0`. A second seed to avoid seed collision. - min_object_covered: An optional `float`. Defaults to `0.1`. + min_object_covered: A Tensor of type `float32`. Defaults to `0.1`. The cropped area of the image must contain at least this fraction of any bounding box supplied. The value of this parameter should be non-negative. In the case of 0, the cropped area does not need to overlap -- GitLab From efcbf6e34e4519172d38be76c08c2d99792fd7be Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 14:05:29 -0700 Subject: [PATCH 525/573] Supported in this CL: * Attaching sharding descriptors to HLO ops * Partitioning the HLO graph into per-device computations based on those sharding descriptors. * All operator support for device placement and ops replicated on all devices. * Elementwise op support for tiled shardings. * 2D Convolution support for tiled shardings (no stride or dilation support). PiperOrigin-RevId: 173946036 --- .../compiler/tf2xla/xla_compilation_device.cc | 7 +- tensorflow/compiler/xla/array.h | 24 +- tensorflow/compiler/xla/client/BUILD | 1 + .../xla/client/computation_builder.cc | 11 +- .../compiler/xla/client/computation_builder.h | 62 ++++- tensorflow/compiler/xla/service/BUILD | 19 ++ .../compiler/xla/service/hlo_computation.cc | 5 +- .../compiler/xla/service/hlo_graph_dumper.cc | 4 +- .../compiler/xla/service/hlo_instruction.cc | 7 +- .../compiler/xla/service/hlo_instruction.h | 35 ++- .../compiler/xla/service/hlo_sharding.cc | 232 ++++++++++++++++++ .../compiler/xla/service/hlo_sharding.h | 165 +++++++++++++ .../compiler/xla/service/hlo_sharding_test.cc | 190 ++++++++++++++ .../xla/service/hlo_tfgraph_builder.cc | 7 +- tensorflow/compiler/xla/service/service.cc | 6 +- .../compiler/xla/service/user_computation.cc | 28 ++- .../compiler/xla/service/user_computation.h | 4 +- .../xla/service/user_computation_test.cc | 17 +- tensorflow/compiler/xla/test_helpers.h | 7 + .../compiler/xla/tools/parser/hlo_lexer.cc | 2 + .../compiler/xla/tools/parser/hlo_parser.cc | 141 ++++++++++- .../xla/tools/parser/hlo_parser_test.cc | 16 +- .../compiler/xla/tools/parser/hlo_token.h | 2 + tensorflow/compiler/xla/xla_data.proto | 30 ++- 24 files changed, 937 insertions(+), 85 deletions(-) create mode 100644 tensorflow/compiler/xla/service/hlo_sharding.cc create mode 100644 tensorflow/compiler/xla/service/hlo_sharding.h create mode 100644 tensorflow/compiler/xla/service/hlo_sharding_test.cc diff --git a/tensorflow/compiler/tf2xla/xla_compilation_device.cc b/tensorflow/compiler/tf2xla/xla_compilation_device.cc index 890a9ccb83..fc866a4c0a 100644 --- a/tensorflow/compiler/tf2xla/xla_compilation_device.cc +++ b/tensorflow/compiler/tf2xla/xla_compilation_device.cc @@ -103,20 +103,17 @@ void XlaCompilationDevice::Compute(OpKernel* op_kernel, DeviceNameUtils::ParseFullName(op_kernel->requested_device(), &parsed), errors::Internal("Unable to parse device name: ", op_kernel->requested_device())); - xla::OpDeviceAssignment assignment; // If no device ID assignment is found, XLA is free to use whatever device it // wants. In practice this usually has the effect of placing things on // device 0. if (parsed.has_id) { - assignment.set_has_device(true); - assignment.set_device(parsed.id); + b->SetSharding(xla::ShardingBuilder::AssignDevice(parsed.id)); } - b->SetDeviceAssignment(assignment); op_kernel->Compute(context); b->ClearOpMetadata(); - b->ClearDeviceAssignment(); + b->ClearSharding(); VLOG(4) << "Done"; } diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h index 2aedafb91f..ba898d1f4e 100644 --- a/tensorflow/compiler/xla/array.h +++ b/tensorflow/compiler/xla/array.h @@ -40,12 +40,13 @@ template class Array { public: // Creates a new array with the specified dimensions. - explicit Array(const std::vector& sizes) : Array(sizes, T()) {} + explicit Array(tensorflow::gtl::ArraySlice sizes) + : Array(sizes, T()) {} // Creates a new array with the specified dimensions and specified value for // every cell. - Array(const std::vector& sizes, T value) - : sizes_(sizes), values_(new T[num_elements()]) { + Array(tensorflow::gtl::ArraySlice sizes, T value) + : sizes_(sizes.begin(), sizes.end()), values_(new T[num_elements()]) { Fill(value); } @@ -192,6 +193,18 @@ class Array { return values_[calculate_index(indexes)]; } + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + const T& operator()(tensorflow::gtl::ArraySlice indexes) const { + return values_[calculate_index(indexes)]; + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + T& operator()(tensorflow::gtl::ArraySlice indexes) { + return values_[calculate_index(indexes)]; + } + // Low-level accessor for stuff like memcmp, handle with care. Returns pointer // to the underlying storage of the array (similarly to std::vector::data()). T* data() const { @@ -218,6 +231,11 @@ class Array { std::multiplies()); } + const T* begin() const { return &values_[0]; } + T* begin() { return &values_[0]; } + const T* end() const { return &values_[num_elements()]; } + T* end() { return &values_[num_elements()]; } + bool operator==(const Array& other) const { if (sizes_.size() != other.sizes_.size()) { return false; diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD index b612698143..f953407a56 100644 --- a/tensorflow/compiler/xla/client/BUILD +++ b/tensorflow/compiler/xla/client/BUILD @@ -170,6 +170,7 @@ cc_library( ":computation", ":global_data", ":padding", + "//tensorflow/compiler/xla:array", "//tensorflow/compiler/xla:array2d", "//tensorflow/compiler/xla:array3d", "//tensorflow/compiler/xla:array4d", diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc index edf5a1822c..24774c4c2a 100644 --- a/tensorflow/compiler/xla/client/computation_builder.cc +++ b/tensorflow/compiler/xla/client/computation_builder.cc @@ -1794,14 +1794,9 @@ StatusOr ComputationBuilder::Build() { void ComputationBuilder::AddCommonFieldsToOpRequest(OpRequest* request) const { *request->mutable_metadata() = metadata_; - *request->mutable_device_assignment() = device_assignment_; -} - -void ComputationBuilder::ClearDeviceAssignment() { device_assignment_.Clear(); } - -void ComputationBuilder::SetDeviceAssignment( - const OpDeviceAssignment& assignment) { - device_assignment_ = assignment; + if (sharding_) { + *request->mutable_sharding() = *sharding_; + } } /* static */ ConvolutionDimensionNumbers diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h index d2f0c7cff0..d282174947 100644 --- a/tensorflow/compiler/xla/client/computation_builder.h +++ b/tensorflow/compiler/xla/client/computation_builder.h @@ -22,6 +22,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/array3d.h" #include "tensorflow/compiler/xla/array4d.h" @@ -42,6 +43,58 @@ limitations under the License. namespace xla { +class ShardingBuilder { + public: + // A shaped array used to describe the assignment of tiles to devices. + using TileAssignment = Array; + + // Creates a replicated sharding - replicate a tensor on every device. + static OpSharding Replicate() { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_REPLICATED); + return result; + } + // Creates a sharding that assigns a tensor to just one device. + static OpSharding AssignDevice(int device) { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + result.add_tile_assignment_dimensions(1); + result.add_tile_assignment_devices(device); + return result; + } + // Creates a tiled sharding with the given tile shape and assignment of tiles + // to devices. + static OpSharding Tile(Shape tile_shape, + const TileAssignment& tile_assignment) { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_OTHER); + for (int64 dim : tile_assignment.dimensions()) { + result.add_tile_assignment_dimensions(dim); + } + for (uint32 device : tile_assignment) { + result.add_tile_assignment_devices(device); + } + return result; + } + // Creates a sharding in one dimension, with the given tile shape which must + // be rank 1 and using devices 0..num_tiles. + static OpSharding Tile1D(Shape tile_shape, int64 num_tiles) { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_OTHER); + + CHECK_EQ(ShapeUtil::Rank(tile_shape), 1); + std::vector dimensions(1, num_tiles); + auto& tile_dimension = (*tile_shape.mutable_dimensions())[0]; + tile_dimension = CeilOfRatio(static_cast(tile_dimension), num_tiles); + *result.mutable_tile_shape() = tile_shape; + result.add_tile_assignment_dimensions(num_tiles); + for (int64 i = 0; i < num_tiles; ++i) { + result.add_tile_assignment_devices(i); + } + return result; + } +}; + // Wraps an XLA client with a convenient interface for building up // computations. Any errors encountered in building up the computation are // deferred from being handled until Build() is called. @@ -78,11 +131,11 @@ class ComputationBuilder { // Sets an OpDeviceAssignment that will be attached to all instructions // until cleared. - void SetDeviceAssignment(const OpDeviceAssignment& assignment); + void SetSharding(const OpSharding& sharding) { sharding_ = sharding; } // Clears the device assignment. Ops will be placed according to the default // placement policy. - void ClearDeviceAssignment(); + void ClearSharding() { sharding_ = tensorflow::gtl::nullopt; } // Sets the builder to a mode where it will die immediately when an error is // encountered, rather than producing it in a deferred fashion when Build() is @@ -894,8 +947,9 @@ class ComputationBuilder { // throughout the TensorFlow op kernel implementations). OpMetadata metadata_; - // Device assignment for the operator. - OpDeviceAssignment device_assignment_; + // Sharding for this operator. This is structured as a "model"-like operation, + // in order to simplify client code, similar to metadata_. + tensorflow::gtl::optional sharding_; TF_DISALLOW_COPY_AND_ASSIGN(ComputationBuilder); }; diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index fe5889efe1..95bc4ca2d9 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -133,6 +133,7 @@ cc_library( "hlo_instruction.cc", "hlo_module.cc", "hlo_opcode.cc", + "hlo_sharding.cc", ], hdrs = [ "dfs_hlo_visitor.h", @@ -141,6 +142,7 @@ cc_library( "hlo_instruction.h", "hlo_module.h", "hlo_opcode.h", + "hlo_sharding.h", ], deps = [ ":hlo_module_config", @@ -148,6 +150,7 @@ cc_library( ":hlo_reachability", ":name_uniquer", ":versioned_computation_handle", + "//tensorflow/compiler/xla:array", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:shape_tree", @@ -238,6 +241,22 @@ tf_cc_test( ], ) +tf_cc_test( + name = "hlo_sharding_test", + srcs = ["hlo_sharding_test.cc"], + deps = [ + ":hlo", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:protobuf_util", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla:test_helpers", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + ], +) + cc_library( name = "call_graph", srcs = ["call_graph.cc"], diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 2285518a0e..72c70b3823 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -56,7 +56,6 @@ std::unique_ptr HloComputation::Builder::Build( HloInstruction* root = root_instruction ? root_instruction : last_added_instruction_; CHECK_NE(nullptr, root); - return WrapUnique(new HloComputation(name_, parameter_count, &instructions_, root, fusion_instruction_)); } @@ -735,6 +734,10 @@ std::unique_ptr HloComputation::Clone(const string& suffix) { } new_instr = instr->CloneWithNewOperands(instr->shape(), new_operands); + new_instr->set_metadata(instr->metadata()); + if (instr->has_sharding()) { + new_instr->set_sharding(instr->sharding()); + } InsertOrDie(&clone_map, instr, new_instr.get()); instructions.push_back(std::move(new_instr)); } diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index e000a06706..11edf49130 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -1039,8 +1039,8 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { if (!opcode_specific_info.empty()) { lines.push_back(opcode_specific_info); } - if (instr->device_assignment().has_device()) { - lines.push_back(StrCat("device=", instr->device_assignment().device())); + if (instr->has_sharding()) { + lines.push_back(StrCat("sharding=", instr->sharding().ToString())); } // Show the shape and layout of the instruction, unless it's an inlined fusion // node -- there the shape and layout is present in the output node. diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 1a03e7ee92..4af52717bb 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -1212,6 +1212,9 @@ std::unique_ptr HloInstruction::Clone( } } clone->set_parent(parent_); + if (has_sharding()) { + clone->set_sharding(sharding()); + } return clone; } @@ -1889,8 +1892,8 @@ std::vector HloInstruction::ExtraAttributesToString() const { if (opcode() == HloOpcode::kGetTupleElement) { extra.push_back(StrCat("index=", tuple_index())); } - if (device_assignment_.has_device()) { - extra.push_back(StrCat("device=", device_assignment_.device())); + if (has_sharding()) { + extra.push_back(StrCat("sharding=", sharding().ToString())); } if (!control_successors_.empty()) { extra.push_back(StrCat( diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index d2a15b0f96..e714d7bc71 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/hlo_sharding.h" #include "tensorflow/compiler/xla/service/name_uniquer.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" @@ -713,6 +714,26 @@ class HloInstruction { fusion_kind_ = kind; } + // Returns the sharding applied to this operator. + // REQUIRES: has_sharding() is true. + const HloSharding& sharding() const { + CHECK(has_sharding()); + return *sharding_; + } + // Returns the sharding applied to this operator, or default_ if none exists. + const HloSharding& sharding_or_default(const HloSharding& default_) const { + return sharding_ ? *sharding_ : default_; + } + // Sets the sharding of this operator. Should only be called by HloModule or + // HloComputation methods. + void set_sharding(const HloSharding& sharding) { + sharding_ = MakeUnique(sharding); + } + // Remove any sharding from this operator. + void clear_sharding() { sharding_ = nullptr; } + // Return true if this operator has a sharding assigned. + bool has_sharding() const { return sharding_ != nullptr; } + // Merges the fused instructions from 'instruction_to_merge' into the // fused instruction set of 'this', updating operands as necessary. // @@ -984,14 +1005,6 @@ class HloInstruction { void RelayoutConstant(const Layout& new_layout, const ShapeIndex& shape_index = {}); - // Gets/sets the device assignment. - const OpDeviceAssignment& device_assignment() const { - return device_assignment_; - } - void set_device_assignment(const OpDeviceAssignment& device_assignment) { - device_assignment_ = device_assignment; - } - private: enum class UseKind { kNoUse, kReuse, kUsePermutingElements, kUse }; @@ -1124,6 +1137,9 @@ class HloInstruction { // The type of the fusion. Used by kFusion only. FusionKind fusion_kind_; + // The sharding, if one exists. + std::unique_ptr sharding_; + // For parameter instructions this field holds the parameter number. int64 parameter_number_ = 0; string parameter_name_; @@ -1184,9 +1200,6 @@ class HloInstruction { // outer-most dimension first). std::vector outer_dimension_partitions_; - // Device assignment for the instruction. - OpDeviceAssignment device_assignment_; - TF_DISALLOW_COPY_AND_ASSIGN(HloInstruction); }; diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc new file mode 100644 index 0000000000..0d019d22f5 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -0,0 +1,232 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_sharding.h" + +#include "tensorflow/core/lib/core/errors.h" + +namespace xla { + +using ::tensorflow::strings::StrCat; + +HloSharding HloSharding::AssignDevice(int64 device_id) { + return HloSharding(device_id); +} + +HloSharding HloSharding::Tile1D(const Shape& input_shape, int64 num_tiles) { + CHECK_EQ(1, ShapeUtil::Rank(input_shape)); + CHECK_GT(num_tiles, 1); + std::vector dimensions(1, num_tiles); + Shape tile_shape = input_shape; + auto& tile_dimension = (*tile_shape.mutable_dimensions())[0]; + tile_dimension = CeilOfRatio(static_cast(tile_dimension), num_tiles); + Array assignment(dimensions); + std::iota(assignment.begin(), assignment.end(), 0); + return HloSharding(tile_shape, assignment); +} + +string HloSharding::ToString() const { + string result = StrCat("{", (replicated_ ? " replicated" : ""), + (maximal_ ? " maximal" : "")); + + if (replicated_) { + return "{replicated}"; + } else if (maximal_) { + return StrCat( + "{maximal device=", static_cast(*tile_assignment_.begin()), "}"); + } else { + return StrCat("{", ShapeUtil::HumanString(tile_shape_), " ", + "devices=", VectorString(tile_assignment_), "}"); + } +} + +bool HloSharding::UsesDevice(int64 device) const { + const auto& devices = tile_assignment_; + return replicated_ || + std::find(devices.begin(), devices.end(), device) != devices.end(); +} + +std::vector HloSharding::TileIndexForDevice(int64 device) const { + CHECK(!ShapeUtil::IsTuple(tile_shape_)); + CHECK(!maximal_); + std::vector ret_index; + tile_assignment_.Each([&](tensorflow::gtl::ArraySlice index, int64 d) { + if (d == device) { + ret_index = {index.begin(), index.end()}; + } + }); + CHECK(!ret_index.empty()); + return ret_index; +} + +int64 HloSharding::DeviceForTileIndex( + tensorflow::gtl::ArraySlice index) const { + CHECK(!replicated_); + if (maximal_) { + return *tile_assignment_.begin(); + } + CHECK_EQ(ShapeUtil::Rank(tile_shape_), tile_assignment_.dimensions().size()); + return tile_assignment_(index); +} + +std::vector HloSharding::TileOffsetForDevice(int64 device) const { + CHECK(!ShapeUtil::IsTuple(tile_shape_)); + + std::vector index = TileIndexForDevice(device); + if (maximal_) { + // Index will always be all zeroes if we're maximal, and tile_shape_ is not + // valid. + return index; + } + for (int64 i = 0; i < index.size(); ++i) { + index[i] *= tile_shape_.dimensions(i); + } + return index; +} + +std::vector HloSharding::TileLimitForDevice(int64 device) const { + CHECK(!ShapeUtil::IsTuple(tile_shape_)); + CHECK(!maximal_); // Maximal shardings do not have a valid tile shape. + + std::vector index = TileIndexForDevice(device); + for (int64 i = 0; i < index.size(); ++i) { + index[i] = (index[i] + 1) * tile_shape_.dimensions(i); + } + return index; +} + +StatusOr HloSharding::UniqueDevice() const { + if (!replicated_ && maximal_) { + return static_cast(*tile_assignment_.begin()); + } + return tensorflow::errors::InvalidArgument( + "UniqueDevice() called on sharding that executes on multiple devices"); +} + +Status HloSharding::Validate(const Shape& shape, int64 num_devices) const { + if (replicated_) { + return Status::OK(); + } + + // All tile assignments must be less than the number of available cores and + // unique. + Status status = Status::OK(); + std::set seen_cores; + tile_assignment_.Each( + [&](tensorflow::gtl::ArraySlice indices, uint32 core) { + // Don't overwrite a bad status, so we report the first error. + if (status.ok()) { + if (core >= num_devices) { + status = + tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "core ", core, " > ", num_devices, " in tile assignment")); + } else if (seen_cores.count(core) != 0) { + status = + tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "core ", core, " is not unique in tile assignment")); + } + } + seen_cores.insert(core); + }); + if (!status.ok()) { + return status; + } + + if (IsTileMaximal()) { + return Status::OK(); + } + + // The tile rank must be the same as the input rank. + if (ShapeUtil::Rank(shape) != ShapeUtil::Rank(tile_shape_)) { + return tensorflow::errors::InvalidArgument( + "Tile rank is different to the input rank"); + } + + // The tile shape must not be the same as the input shape without maximal_ + // also set. If this is the case, we're not actually sharded and the correct + // constructor should have been used. + if (ShapeUtil::Equal(shape, tile_shape_)) { + return tensorflow::errors::InvalidArgument( + "Tile shape is the same as the input shape. If a replicated sharding " + "was intended, use HloSharding::Replicated(). If a device placement " + "was intended, use HloSharding::AssignDevice()"); + } + + // The tile shape must not be greater than the input shape in any dimension. + for (int64 i = 0, e = ShapeUtil::Rank(shape); i != e; ++i) { + auto tile_dim = tile_shape_.dimensions(i); + auto shape_dim = shape.dimensions(i); + if (tile_dim > shape_dim) { + return tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "Tile is larger than input shape (dimension ", i, ", ", tile_dim, + " > ", shape_dim)); + } + } + + // The tile assignment tensor must be exactly dimensioned to ceil(shape[dim] + // tile[dim]) for every dimension contained within tile. + for (int64 i = 0, e = tile_assignment_.dimensions().size(); i != e; ++i) { + int64 expected_dim = + CeilOfRatio(shape.dimensions(i), tile_shape_.dimensions(i)); + if (tile_assignment_.dimensions()[i] != expected_dim) { + return tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "Tile assignment tensor has incorrect shape. Dimension ", i, + " expected ", expected_dim, " but got ", + tile_assignment_.dimensions()[i])); + } + } + + return Status::OK(); +} + +/*static*/ StatusOr HloSharding::FromProto( + const OpSharding& proto) { + if (proto.type() == OpSharding::Type::OpSharding_Type_REPLICATED) { + return Replicate(); + } else if (proto.type() == OpSharding::Type::OpSharding_Type_MAXIMAL) { + return HloSharding(proto.tile_assignment_devices(0)); + } + // Some versions of gcc cannot infer the TileAssignment constructor from a + // braced initializer-list, so create one manually. + std::vector devices(proto.tile_assignment_devices().begin(), + proto.tile_assignment_devices().end()); + Array tile_assignment( + std::vector(proto.tile_assignment_dimensions().begin(), + proto.tile_assignment_dimensions().end())); + std::copy(proto.tile_assignment_devices().begin(), + proto.tile_assignment_devices().end(), tile_assignment.begin()); + return HloSharding(proto.tile_shape(), tile_assignment); +} + +OpSharding HloSharding::ToProto() const { + OpSharding result; + *result.mutable_tile_shape() = tile_shape_; + for (int64 dim : tile_assignment_.dimensions()) { + result.add_tile_assignment_dimensions(dim); + } + for (auto device : tile_assignment_) { + result.add_tile_assignment_devices(device); + } + if (IsReplicated()) { + result.set_type(OpSharding::Type::OpSharding_Type_REPLICATED); + } else if (IsTileMaximal()) { + result.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + } else { + result.set_type(OpSharding::Type::OpSharding_Type_OTHER); + } + return result; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h new file mode 100644 index 0000000000..d7ada30c70 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_sharding.h @@ -0,0 +1,165 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// HLO shardings describe how an HLO instruction is split across multiple +// computations. + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_ + +#include + +#include "tensorflow/compiler/xla/array.h" +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/protobuf_util.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/lib/hash/hash.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +// HLO shardings describe how an HLO instruction is split across multiple +// computations. +class HloSharding { + public: + // Creates a trivial sharding that replicates a maximal tile across all + // devices. + static HloSharding Replicate() { return HloSharding(); } + + // Creates a sharding that emulates device placement; a tile shape equal to + // the input shape (one tile) assigned to a single device. + static HloSharding AssignDevice(int64 device_id); + + // Creates a new sharding which splits a shape into tiles each with shape + // `tile_shape`. Each tile is assigned to one device, which is specified by + // `tile_assignment`. Any tensor not a multiple of the tile size in any + // dimension is implicitly padded to the tile size. + // + // e.g. Tile({2, 2}, {0, 1}) on a tensor of shape {3, 2} would look like: + // 2 1 padding + // <------><-> + // +----+----+ + // | 0 | 1 | + // +----+----+ + // + // Split into two tiles, one of which is implicitly padded by one. + static HloSharding Tile(const Shape& tile_shape, + const Array& tile_assignment) { + return HloSharding(tile_shape, tile_assignment); + } + + // Creates a new sharding which splits a one-dimensional input shape into + // `num_tiles` tiles. + static HloSharding Tile1D(const Shape& input_shape, int64 num_tiles); + + // Create a new sharding from a protobuf OpSharding. + static StatusOr FromProto(const OpSharding& proto); + + OpSharding ToProto() const; + string ToString() const; + + // Validate that this sharding can be applied to a tensor with shape `shape`. + Status Validate(const Shape& shape, int64 num_devices) const; + + // Returns true if the sharding is trivial: replicate on all devices. + bool IsReplicated() const { return replicated_; } + + // Returns true if the tile size is the same as the input size. + bool IsTileMaximal() const { return maximal_; } + + // Returns true if the sharding defines an operation on the given device. + bool UsesDevice(int64 device) const; + + // Returns the tile that should be executed on the given device. + std::vector TileIndexForDevice(int64 device) const; + + // Returns the device that should execute the given tile. + // It is an error to call this if is_replicated() is true. + int64 DeviceForTileIndex(tensorflow::gtl::ArraySlice index) const; + + // Given a device ID, returns the offset within the input space of the + // tile that should be executed on the given core. This returns the lower + // extent of the tile in the input space. + std::vector TileOffsetForDevice(int64 device) const; + + // Given a device ID, returns the limit within the input space of the + // tile that should be executed on the given core. This returns the upper + // extent of the tile in the input space. + std::vector TileLimitForDevice(int64 device) const; + + // Returns the single device this op operates on. + // Requires !Replicated() && IsTileMaximal(). + StatusOr UniqueDevice() const; + + // Returns true if this op only uses a single device. + bool HasUniqueDevice() const { return !IsReplicated() && IsTileMaximal(); } + + bool operator==(const HloSharding& other) const { + return replicated_ == other.replicated_ && maximal_ == other.maximal_ && + protobuf_util::ProtobufEquals(tile_shape_, other.tile_shape_) && + tile_assignment_ == other.tile_assignment_; + } + bool operator!=(const HloSharding& other) const { return !(*this == other); } + + size_t Hash() const { + if (replicated_) { + return 0; + } + size_t h = 0; + for (uint32 v : tile_assignment_) { + h = tensorflow::Hash64Combine(h, std::hash{}(v)); + } + for (uint32 v : tile_shape_.dimensions()) { + h = tensorflow::Hash64Combine(h, std::hash{}(v)); + } + return h; + } + + // Gets the tile shape. + // It is an error to call this if IsTileMaximal() is true. + const Shape& tile_shape() const { return tile_shape_; } + // Gets the tile assignment tensor. + // It is an error to call this if IsReplicated() is true. + const Array& tile_assignment() const { return tile_assignment_; } + + private: + HloSharding() + : replicated_(true), + maximal_(true), + tile_shape_(), + tile_assignment_({0}) {} + explicit HloSharding(int64 device_id) + : replicated_(false), + maximal_(true), + tile_shape_(), + tile_assignment_({1}, device_id) {} + HloSharding(const Shape& tile_shape, const Array& tile_assignment) + : replicated_(false), + maximal_(false), + tile_shape_(tile_shape), + tile_assignment_(tile_assignment) {} + + bool replicated_; + bool maximal_; + Shape tile_shape_; + Array tile_assignment_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_ diff --git a/tensorflow/compiler/xla/service/hlo_sharding_test.cc b/tensorflow/compiler/xla/service/hlo_sharding_test.cc new file mode 100644 index 0000000000..d0a20471a0 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_sharding_test.cc @@ -0,0 +1,190 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_sharding.h" + +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/compiler/xla/test_helpers.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/util.h" + +namespace xla { +namespace { + +Array MakeArray(tensorflow::gtl::ArraySlice dimensions, + tensorflow::gtl::ArraySlice contents) { + Array a(dimensions); + std::copy(contents.begin(), contents.end(), a.begin()); + return a; +} + +class HloShardingTest : public HloTestBase {}; + +TEST_F(HloShardingTest, Replicate) { + Shape tile_shape = ShapeUtil::MakeShape(U32, {4}); + HloSharding sharding = HloSharding::Replicate(); + EXPECT_TRUE(sharding.IsReplicated()); + EXPECT_TRUE(sharding.IsTileMaximal()); + EXPECT_TRUE(sharding.UsesDevice(0)); + EXPECT_TRUE(sharding.UsesDevice(65535)); + + HloSharding other = HloSharding::Replicate(); + EXPECT_EQ(other, sharding); + + EXPECT_IS_OK(sharding.Validate(ShapeUtil::MakeShape(U32, {4}), + /*num_devices=*/2)); + EXPECT_IS_NOT_OK(sharding.UniqueDevice()); +} + +TEST_F(HloShardingTest, DevicePlacement) { + HloSharding sharding = HloSharding::AssignDevice(5); + EXPECT_FALSE(sharding.IsReplicated()); + EXPECT_TRUE(sharding.IsTileMaximal()); + EXPECT_FALSE(sharding.UsesDevice(0)); + EXPECT_TRUE(sharding.UsesDevice(5)); + EXPECT_EQ(5, sharding.UniqueDevice().ValueOrDie()); + + HloSharding other = HloSharding::Replicate(); + EXPECT_NE(other, sharding); + + EXPECT_IS_OK(sharding.Validate(ShapeUtil::MakeShape(U32, {4}), + /*num_devices=*/6)); + EXPECT_IS_NOT_OK( + sharding.Validate(ShapeUtil::MakeShape(U32, {4}), /*num_devices=*/5)); +} + +TEST_F(HloShardingTest, Tile) { + { + // Test should fail because of a duplicate tile assignment. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 0, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {4, 6}), + /*num_devices=*/4)); + } + + { + // Test should pass. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 1, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(U32, {4, 6}), + /*num_devices=*/2)); + } + + { + // Test should fail due to the tile being larger than the input space. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 1, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {2, 2}), + /*num_devices=*/4)); + } + + { + // Test should fail due to the tile not dividing the input space into 4 + // sections (even with padding). + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 1, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {6, 3}), + /*num_devices=*/4)); + } + + { + // Test should pass. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + EXPECT_IS_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {3, 5}), + /*num_devices=*/5)); + + EXPECT_EQ(0, sharding.DeviceForTileIndex({0, 0})); + EXPECT_EQ(3, sharding.DeviceForTileIndex({0, 1})); + EXPECT_EQ(2, sharding.DeviceForTileIndex({1, 0})); + EXPECT_EQ(1, sharding.DeviceForTileIndex({1, 1})); + + EXPECT_EQ(sharding.TileOffsetForDevice(0), (std::vector{0, 0})); + EXPECT_EQ(sharding.TileOffsetForDevice(3), (std::vector{0, 3})); + EXPECT_EQ(sharding.TileOffsetForDevice(2), (std::vector{2, 0})); + EXPECT_EQ(sharding.TileOffsetForDevice(1), (std::vector{2, 3})); + + EXPECT_IS_NOT_OK(sharding.UniqueDevice()); + } +} + +TEST_F(HloShardingTest, Hash) { + auto hash_compare_equal = [](const HloSharding& a, const HloSharding& b) { + if (a.Hash() != b.Hash()) { + return false; + } + return a == b; + }; + + { + HloSharding sharding1 = HloSharding::Replicate(); + HloSharding sharding2 = HloSharding::Replicate(); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + HloSharding sharding1 = HloSharding::AssignDevice(1); + HloSharding sharding2 = HloSharding::AssignDevice(1); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + HloSharding sharding1 = HloSharding::AssignDevice(1); + HloSharding sharding2 = HloSharding::AssignDevice(2); + EXPECT_FALSE(hash_compare_equal(sharding1, sharding2)); + } + + { + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding1 = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + HloSharding sharding2 = HloSharding::Tile(ShapeUtil::MakeShape(U32, {2, 3}), + MakeArray({2, 2}, {0, 3, 2, 1})); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding1 = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + HloSharding sharding2 = HloSharding::Tile(ShapeUtil::MakeShape(U32, {2, 3}), + MakeArray({2, 2}, {0, 3, 2, 1})); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding1 = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + HloSharding sharding2 = HloSharding::Tile(ShapeUtil::MakeShape(U32, {2, 3}), + MakeArray({2, 2}, {0, 3, 1, 2})); + EXPECT_FALSE(hash_compare_equal(sharding1, sharding2)); + } +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc index 2007a8f11d..06abe00747 100644 --- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc +++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc @@ -198,9 +198,10 @@ Status HloTfGraphBuilder::AddInstruction(const HloInstruction* instruction) { NodeDef* node_def = graph_def_.add_node(); node_def->set_name(GetNodeNameForInstruction(instruction)); node_def->set_op(GetOpDefName(instruction)); - if (instruction->device_assignment().has_device()) { - node_def->set_device( - GetDeviceName(instruction->device_assignment().device())); + if (instruction->has_sharding() && + instruction->sharding().HasUniqueDevice()) { + TF_ASSIGN_OR_RETURN(int64 device, instruction->sharding().UniqueDevice()); + node_def->set_device(GetDeviceName(device)); } SetNodeAttrs(instruction, node_def); if (instruction->opcode() == HloOpcode::kFusion) { diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 0fbc2f2fec..bac33d8102 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -1415,9 +1415,9 @@ tensorflow::Status Service::Op(const OpRequest* arg, OpResponse* result) { // proto in the above switch statement. TF_ASSIGN_OR_RETURN(ComputationDataHandle handle, handle_status); TF_RETURN_IF_ERROR(computation->SetOpMetadata(handle, arg->metadata())); - TF_RETURN_IF_ERROR( - computation->SetOpDeviceAssignment(handle, arg->device_assignment())); - + if (arg->has_sharding()) { + TF_RETURN_IF_ERROR(computation->SetOpSharding(handle, arg->sharding())); + } return tensorflow::Status::OK(); } diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index adf7972e0d..0bdeffaf25 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -1315,20 +1315,19 @@ Status UserComputation::SetOpMetadata(const ComputationDataHandle& handle, return Status::OK(); } -Status UserComputation::SetOpDeviceAssignment( - const ComputationDataHandle& handle, - const OpDeviceAssignment& device_assignment) { +Status UserComputation::SetOpSharding(const ComputationDataHandle& handle, + const OpSharding& sharding) { tensorflow::mutex_lock lock(mutex_); int64 handle_value = handle.handle(); if (session_computation_.requests().count(handle_value) == 0) { - return InvalidArgument("Invalid handle in SetOpDeviceAssignment (%lld)", + return InvalidArgument("Invalid handle in SetOpSharding (%lld)", handle_value); } *session_computation_.mutable_requests() ->at(handle_value) .mutable_request() - ->mutable_device_assignment() = device_assignment; + ->mutable_sharding() = sharding; return Status::OK(); } @@ -2518,7 +2517,9 @@ HloInstruction* ComputationLowerer::ImplicitBroadcastToExplicitBroadcast( if (ShapeUtil::IsScalar(operand->shape())) { HloInstruction* broadcast = hlo_builder_.AddInstruction( HloInstruction::CreateBroadcast(broadcast_shape, operand, {})); - broadcast->set_device_assignment(operand->device_assignment()); + if (operand->has_sharding()) { + broadcast->set_sharding(operand->sharding()); + } return broadcast; } // Do explicit broadcast for degenerate broadcast. @@ -2536,12 +2537,16 @@ HloInstruction* ComputationLowerer::ImplicitBroadcastToExplicitBroadcast( ShapeUtil::MakeShape(operand->shape().element_type(), reshaped_dimensions), operand)); - reshaped_operand->set_device_assignment(operand->device_assignment()); + if (operand->has_sharding()) { + reshaped_operand->set_sharding(operand->sharding()); + } // Broadcast 'reshape' up to the larger size. HloInstruction* broadcast = hlo_builder_.AddInstruction(HloInstruction::CreateBroadcast( broadcast_shape, reshaped_operand, broadcast_dimensions)); - broadcast->set_device_assignment(operand->device_assignment()); + if (operand->has_sharding()) { + broadcast->set_sharding(operand->sharding()); + } return broadcast; } @@ -2556,8 +2561,11 @@ void ComputationLowerer::Visit( HloInstruction* hlo_instruction = hlo_builder_.AddInstruction(std::move(instruction)); hlo_instruction->set_metadata(request.request().metadata()); - hlo_instruction->set_device_assignment( - request.request().device_assignment()); + if (request.request().has_sharding()) { + OpSharding op_sharding = request.request().sharding(); + hlo_instruction->set_sharding( + HloSharding::FromProto(op_sharding).ValueOrDie()); + } return hlo_instruction; }; auto lookup_instruction = [&](const ComputationDataHandle& handle) { diff --git a/tensorflow/compiler/xla/service/user_computation.h b/tensorflow/compiler/xla/service/user_computation.h index 6f3bf430fc..dabf68e298 100644 --- a/tensorflow/compiler/xla/service/user_computation.h +++ b/tensorflow/compiler/xla/service/user_computation.h @@ -262,8 +262,8 @@ class UserComputation { const OpMetadata& metadata); // Sets the device assignment on the Hlo instruction referenced by 'handle'. - Status SetOpDeviceAssignment(const ComputationDataHandle& handle, - const OpDeviceAssignment& device_assignment); + Status SetOpSharding(const ComputationDataHandle& handle, + const OpSharding& sharding); // Builds a HLO computation from the UserComputation. The parameter "resolver" // is a function which returns a pointer to the HloComputation corresponding diff --git a/tensorflow/compiler/xla/service/user_computation_test.cc b/tensorflow/compiler/xla/service/user_computation_test.cc index 43a857935a..5afaf226ae 100644 --- a/tensorflow/compiler/xla/service/user_computation_test.cc +++ b/tensorflow/compiler/xla/service/user_computation_test.cc @@ -224,10 +224,13 @@ TEST_F(UserComputationTest, CheckImplicitBroadcastToExplicitBroadcast) { TF_ASSERT_OK_AND_ASSIGN(ComputationDataHandle b_handle, computation.AddParameterInstruction(b_request)); - OpDeviceAssignment assignment; - assignment.set_has_device(true); - assignment.set_device(7); - TF_EXPECT_OK(computation.SetOpDeviceAssignment(b_handle, assignment)); + const int64 kDevice = 7; + OpSharding sharding; + sharding.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + sharding.add_tile_assignment_dimensions(1); + sharding.add_tile_assignment_devices(kDevice); + + TF_EXPECT_OK(computation.SetOpSharding(b_handle, sharding)); BinaryOpRequest add; add.set_binop(BINOP_ADD); @@ -260,12 +263,10 @@ TEST_F(UserComputationTest, CheckImplicitBroadcastToExplicitBroadcast) { const HloInstruction* broadcast = hlo_computation->root_instruction()->operand(1); - EXPECT_TRUE(broadcast->device_assignment().has_device()); - EXPECT_EQ(assignment.device(), broadcast->device_assignment().device()); + EXPECT_TRUE(broadcast->has_sharding()); const HloInstruction* reshape = broadcast->operand(0); - EXPECT_TRUE(reshape->device_assignment().has_device()); - EXPECT_EQ(assignment.device(), reshape->device_assignment().device()); + EXPECT_TRUE(reshape->has_sharding()); } TEST_F(UserComputationTest, EliminateDegenerateBroadcastAfterIndimBroadcast) { diff --git a/tensorflow/compiler/xla/test_helpers.h b/tensorflow/compiler/xla/test_helpers.h index 634cdb5aa2..17bae2e4f6 100644 --- a/tensorflow/compiler/xla/test_helpers.h +++ b/tensorflow/compiler/xla/test_helpers.h @@ -62,9 +62,16 @@ inline const ::tensorflow::Status& GetStatus(const StatusOr& status) { #define EXPECT_IS_OK(expression) \ EXPECT_EQ(tensorflow::Status::OK(), \ xla::testing::internal_status::GetStatus(expression)) +#define EXPECT_IS_NOT_OK(expression) \ + EXPECT_NE(tensorflow::Status::OK(), \ + xla::testing::internal_status::GetStatus(expression)) #undef ASSERT_IS_OK #define ASSERT_IS_OK(expression) \ ASSERT_EQ(tensorflow::Status::OK(), \ xla::testing::internal_status::GetStatus(expression)) +#undef ASSERT_IS_NOT_OK +#define ASSERT_IS_NOT_OK(expression) \ + ASSERT_NE(tensorflow::Status::OK(), \ + xla::testing::internal_status::GetStatus(expression)) #endif // TENSORFLOW_COMPILER_XLA_TEST_HELPERS_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc index fba343de48..486df68540 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc @@ -204,6 +204,8 @@ TokKind HloLexer::LexIdentifier() { KEYWORD(HloModule); KEYWORD(ENTRY); KEYWORD(ROOT); + KEYWORD(maximal); + KEYWORD(replicated); #undef KEYWORD diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc index d91404d73a..7c1eaa9f7f 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -49,6 +49,7 @@ class HloParser { bool ParseInstructionList(HloComputation::Builder* builder, string* root_name); bool ParseInstruction(HloComputation::Builder* builder, string* root_name); + bool ParseSharding(HloInstruction* instruction); bool ParseLiteral(std::unique_ptr* literal, const Shape& shape); bool ParseOperands(std::vector* operands); // Fill parsed operands into 'operands' and expect a certain number of @@ -409,21 +410,147 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, return TokenError(StrCat("parsing not yet implemented for op: ", HloOpcodeString(opcode))); } - // Parse "device=". + // Parse "sharding=". if (lexer_.GetKind() == TokKind::kComma) { - int64 device; - if (!ParseExtraAttribute(&device, /*expected_attribute=*/"device")) { + if (!ParseSharding(instruction)) { return false; } - OpDeviceAssignment assignment; - assignment.set_has_device(true); - assignment.set_device(device); - instruction->set_device_assignment(assignment); } return AddInstruction(name, instruction); } +// ::= '{' 'replicated'? 'maximal'? ('device=' int)? shape? ('devices=' ('[' +// dims ']')* device_list)? '}' dims ::= int_list device_list ::= int_list +bool HloParser::ParseSharding(HloInstruction* instruction) { + if (!ParseToken(TokKind::kComma, + "expects ',' in front of an extra attribute")) { + return false; + } + string attribute_name; + if (!ParseAttributeName(&attribute_name) || attribute_name != "sharding") { + return TokenError("expects attribute name: sharding"); + } + + if (!ParseToken(TokKind::kLbrace, + "expected '{' to start sharding attribute")) { + return false; + } + + bool maximal = false; + bool replicated = false; + std::vector devices; + std::vector tile_assignment_dimensions; + Shape tile_shape; + while (lexer_.GetKind() != TokKind::kRbrace) { + switch (lexer_.GetKind()) { + case TokKind::kw_maximal: + maximal = true; + lexer_.Lex(); + break; + case TokKind::kw_replicated: + replicated = true; + lexer_.Lex(); + break; + case TokKind::kAttributeName: { + if (lexer_.GetStrVal() == "device") { + if (lexer_.Lex() != TokKind::kInt) { + return TokenError("device= attribute must be an integer"); + } + devices = {lexer_.GetInt64Val()}; + lexer_.Lex(); + } else if (lexer_.GetStrVal() == "devices") { + lexer_.Lex(); + if (!ParseToken(TokKind::kLsquare, + "expected '[' to start sharding devices shape")) { + return false; + } + + do { + int64 dim; + if (!ParseInt64(&dim)) { + return false; + } + tile_assignment_dimensions.push_back(dim); + } while (EatIfPresent(TokKind::kComma)); + + if (!ParseToken(TokKind::kRsquare, + "expected ']' to start sharding devices shape")) { + return false; + } + do { + int64 device; + if (!ParseInt64(&device)) { + return false; + } + devices.push_back(device); + } while (EatIfPresent(TokKind::kComma)); + } else { + return TokenError( + "unknown attribute in sharding: expected device= or devices="); + } + break; + } + case TokKind::kShape: + tile_shape = lexer_.GetShapeVal(); + lexer_.Lex(); + break; + case TokKind::kRbrace: + break; + default: + return TokenError("unexpected token"); + } + } + + OpSharding sharding; + if (replicated) { + if (!devices.empty()) { + return TokenError( + "replicated shardings should not have any devices assigned"); + } + if (!ShapeUtil::Equal(tile_shape, Shape())) { + return TokenError( + "replicated shardings should not have any tile shape set"); + } + sharding.set_type(OpSharding::Type::OpSharding_Type_REPLICATED); + } else if (maximal) { + if (devices.size() != 1) { + return TokenError( + "maximal shardings should have exactly one device assigned"); + } + if (!ShapeUtil::Equal(tile_shape, Shape())) { + return TokenError("maximal shardings should not have any tile shape set"); + } + sharding.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + sharding.add_tile_assignment_devices(devices[0]); + } else { + if (devices.size() <= 1) { + return TokenError( + "non-maximal shardings must have more than one device assigned"); + } + if (ShapeUtil::Equal(tile_shape, Shape())) { + return TokenError("non-maximal shardings should have a tile shape set"); + } + if (tile_assignment_dimensions.empty()) { + return TokenError( + "non-maximal shardings must have a tile assignment list including " + "dimensions"); + } + sharding.set_type(OpSharding::Type::OpSharding_Type_OTHER); + *sharding.mutable_tile_shape() = tile_shape; + for (int64 dim : tile_assignment_dimensions) { + sharding.add_tile_assignment_dimensions(dim); + } + for (int64 device : devices) { + sharding.add_tile_assignment_devices(device); + } + } + + instruction->set_sharding(HloSharding::FromProto(sharding).ValueOrDie()); + lexer_.Lex(); + return true; +} + bool HloParser::ParseLiteral(std::unique_ptr* literal, const Shape& shape) { switch (shape.element_type()) { diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc index 2bf1cce1c0..5be4d6a2cb 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -100,9 +100,9 @@ ENTRY %add_constants () -> f32[] { R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module: ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { - %v1 = f32[4]{0} parameter(0), device=1 - %v2 = f32[4]{0} parameter(1), device=1 - %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2) + %v1 = f32[4]{0} parameter(0), sharding={maximal device=1} + %v2 = f32[4]{0} parameter(1), sharding={maximal device=1} + %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2), sharding={replicated} ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) } @@ -164,9 +164,9 @@ ENTRY %WhileWithScalarS32Result.v2 () -> s32[] { R"(HloModule TwoSendRecvBothWayRecvFist_module: ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> f32[] { - %recv = f32[] recv(), channel_id=15, device=1 - ROOT %constant = f32[] constant(2.1), device=0 - %send = () send(f32[] %constant), channel_id=16, device=0 + %recv = f32[] recv(), channel_id=15, sharding={maximal device=1} + ROOT %constant = f32[] constant(2.1), sharding={maximal device=0} + %send = () send(f32[] %constant), channel_id=16, sharding={maximal device=0} } )" @@ -180,7 +180,7 @@ ENTRY %GetTupleElement.v4 () -> s32[] { %constant = f32[] constant(1.23) %constant.1 = s32[] constant(4) %tuple = (f32[], s32[]) tuple(f32[] %constant, s32[] %constant.1) - ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1, device=0 + ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1, sharding={maximal device=0} } )" @@ -289,7 +289,7 @@ TEST_F(HloParserTest, MoreConstants) { ENTRY %SelectScalarS32True.v4 () -> s32[] { %constant.2 = pred[] constant(true) - %constant.1 = s32[] constant(-42) + %constant.1 = s32[] constant(-42), sharding={s32[5,6] devices=[2,3]1,2,3,4} %constant = s32[] constant(42) %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant) } diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h index 1d56ea3478..a40300e2bf 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_token.h +++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h @@ -44,6 +44,8 @@ enum class TokKind { kw_ROOT, kw_true, kw_false, + kw_maximal, + kw_replicated, // Typed tokens. kName, // %foo diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index fe47f85c12..2a8dc682a1 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -812,18 +812,32 @@ message RecvRequest { ChannelHandle channel_handle = 2; } -message OpDeviceAssignment { - bool has_device = 1; - - // Number of the device to which this operator is assigned. Ignored if - // 'has_device' is false. - int32 device = 2; +message OpSharding { + enum Type { + // This sharding is replicated across all devices (implies maximal, + // all other fields are unused). + REPLICATED = 0; + // This sharding is maximal - one device runs the entire operation. + MAXIMAL = 1; + // Neither of the above; tile_shape and tile_assignment are both used. + OTHER = 2; + } + Type type = 1; + // The shape of the sharded tile. + Shape tile_shape = 2; + // The shape of the tile assignment tensor - this must be the same rank as + // tile_shape and the product of its dimensions must equal + // tile_assignment_devices.size(). + repeated int64 tile_assignment_dimensions = 3; + // Flattened list of device IDs. The order of flattening is the same as used + // by IndexUtil::MultiToLinearIndex(tile_assignment_shape). + repeated int64 tile_assignment_devices = 4; } message OpRequest { ComputationHandle computation = 1; OpMetadata metadata = 33; - OpDeviceAssignment device_assignment = 39; + OpSharding sharding = 40; oneof op { BinaryOpRequest binary_op_request = 2; @@ -862,7 +876,7 @@ message OpRequest { BatchNormTrainingRequest batch_norm_training_request = 35; BatchNormGradRequest batch_norm_grad_request = 37; BatchNormInferenceRequest batch_norm_inference_request = 38; - // Next: 40 + // Next: 41 } } -- GitLab From 3639aa7ff1e40648aebf2e45dca60d3d798586d5 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 30 Oct 2017 14:10:55 -0700 Subject: [PATCH 526/573] Always run iterator deleter in eager mode for safety. PiperOrigin-RevId: 173947019 --- tensorflow/contrib/eager/python/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py index f83c470411..357e3420d2 100644 --- a/tensorflow/contrib/eager/python/datasets.py +++ b/tensorflow/contrib/eager/python/datasets.py @@ -84,7 +84,7 @@ class Iterator(object): def __del__(self): if self._resource is not None: - with ops.device("/device:CPU:0"): + with ops.device("/device:CPU:0"), context.eager_mode(): resource_variable_ops.destroy_resource_op(self._resource) self._resource = None -- GitLab From 1d6dae88efef68dd7fbeeb5c39ea0f69c1c721c1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 14:16:18 -0700 Subject: [PATCH 527/573] Add check to tf.device when called with a function in eager mode. PiperOrigin-RevId: 173947845 --- tensorflow/python/framework/ops.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index b5e3e548bd..e68eac3723 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -4339,11 +4339,18 @@ def device(device_name_or_function): Returns: A context manager that specifies the default device to use for newly created ops. + + Raises: + RuntimeError: If eager execution is enabled and a function is passed in. """ if context.in_graph_mode(): return get_default_graph().device(device_name_or_function) else: # TODO(agarwal): support device functions in EAGER mode. + if callable(device_name_or_function): + raise RuntimeError( + "tf.device does not support functions when eager execution " + "is enabled.") return context.device(device_name_or_function) -- GitLab From 25620825bc1c9131c29ba5abeba8ee3b1d18e911 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 14:29:15 -0700 Subject: [PATCH 528/573] Dataset: Adds eager warnings to make_initializable_iterator and make_one_shot_iterator. PiperOrigin-RevId: 173949737 --- tensorflow/python/data/ops/dataset_ops.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 151556994f..343f316281 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.data.ops import iterator_ops from tensorflow.python.data.util import nest +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import function @@ -80,7 +81,14 @@ class Dataset(object): Returns: An `Iterator` over the elements of this dataset. + + Raises: + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "dataset.make_initializable_iterator is not supported when eager " + "execution is enabled.") if shared_name is None: shared_name = "" iterator_resource = gen_dataset_ops.iterator( @@ -102,7 +110,14 @@ class Dataset(object): Returns: An `Iterator` over the elements of this dataset. + + Raises: + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "dataset.make_one_shot_iterator is not supported when eager " + "execution is enabled.") # NOTE(mrry): We capture by value here to ensure that `_make_dataset()` is # a 0-argument function. @function.Defun(capture_by_value=True) -- GitLab From e40eb810a6be3dbf94e95724a16c2344060523c4 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Mon, 30 Oct 2017 14:30:51 -0700 Subject: [PATCH 529/573] TFE: Add errors for classic tf.summary.* ops and FileWriter PiperOrigin-RevId: 173949980 --- tensorflow/contrib/eager/python/BUILD | 1 + tensorflow/contrib/eager/python/tfe_test.py | 26 +++++++++++++++++ tensorflow/python/BUILD | 1 + tensorflow/python/summary/summary.py | 32 ++++++++++++++++++++- tensorflow/python/summary/writer/writer.py | 14 +++++++++ 5 files changed, 73 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 96393f9f5a..614a080e61 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -41,6 +41,7 @@ cuda_py_test( "//tensorflow/python:math_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:platform_test", + "//tensorflow/python:summary", ], ) diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index d8a38923a3..0dedb2fd7c 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -17,6 +17,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import tempfile + from tensorflow.contrib.eager.python import tfe from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors @@ -27,6 +29,8 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import numerics from tensorflow.python.ops import variables from tensorflow.python.platform import test +from tensorflow.python.summary import summary +from tensorflow.python.summary.writer import writer class TFETest(test_util.TensorFlowTestCase): @@ -108,6 +112,28 @@ class TFETest(test_util.TensorFlowTestCase): r'add_check_numerics_ops\(\) is not compatible with eager execution'): numerics.add_check_numerics_ops() + def testClassicSummaryOpsErrorOut(self): + x = constant_op.constant(42) + x_summary = summary.scalar('x', x) + y = constant_op.constant([1, 3, 3, 7]) + y_summary = summary.histogram('hist', y) + + with self.assertRaisesRegexp( + RuntimeError, + r'Merging tf\.summary\.\* ops is not compatible with eager execution'): + summary.merge([x_summary, y_summary]) + + with self.assertRaisesRegexp( + RuntimeError, + r'Merging tf\.summary\.\* ops is not compatible with eager execution'): + summary.merge_all() + + def testClassicSummaryFileWriterErrorsOut(self): + with self.assertRaisesRegexp( + RuntimeError, + r'tf\.summary\.FileWriter is not compatible with eager execution'): + writer.FileWriter(tempfile.mkdtemp()) + if __name__ == '__main__': tfe.enable_eager_execution() diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e167af96d0..02e88f4888 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3792,6 +3792,7 @@ py_library( ":summary_op_util", ":summary_ops", ":util", + "//tensorflow/python/eager:context", "//third_party/py/numpy", "@six_archive//:six", ], diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py index 90afcc0a11..355593eca5 100644 --- a/tensorflow/python/summary/summary.py +++ b/tensorflow/python/summary/summary.py @@ -48,6 +48,7 @@ from tensorflow.core.util.event_pb2 import SessionLog from tensorflow.core.util.event_pb2 import TaggedRunMetadata # pylint: enable=unused-import +from tensorflow.python.eager import context as _context from tensorflow.python.framework import dtypes as _dtypes from tensorflow.python.framework import ops as _ops from tensorflow.python.ops import gen_logging_ops as _gen_logging_ops @@ -263,8 +264,20 @@ def merge(inputs, collections=None, name=None): Returns: A scalar `Tensor` of type `string`. The serialized `Summary` protocol buffer resulting from the merging. + + Raises: + RuntimeError: If called with eager mode enabled. + + @compatibility(eager) + Not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ # pylint: enable=line-too-long + if _context.in_eager_mode(): + raise RuntimeError( + 'Merging tf.summary.* ops is not compatible with eager execution. ' + 'Use tf.contrib.summary instead.') name = _summary_op_util.clean_tag(name) with _ops.name_scope(name, 'Merge', inputs): # pylint: disable=protected-access @@ -284,7 +297,19 @@ def merge_all(key=_ops.GraphKeys.SUMMARIES): If no summaries were collected, returns None. Otherwise returns a scalar `Tensor` of type `string` containing the serialized `Summary` protocol buffer resulting from the merging. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ + if _context.in_eager_mode(): + raise RuntimeError( + 'Merging tf.summary.* ops is not compatible with eager execution. ' + 'Use tf.contrib.summary instead.') summary_ops = _ops.get_collection(key) if not summary_ops: return None @@ -306,6 +331,11 @@ def get_summary_description(node_def): Raises: ValueError: if the node is not a summary op. + + @compatibility(eager) + Not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ if node_def.op != 'TensorSummary': @@ -317,7 +347,7 @@ def get_summary_description(node_def): _allowed_symbols = [ - 'Summary', 'SummaryDescription', 'Event', 'TaggedRunMetadata', 'SessionLog' + 'Summary', 'SummaryDescription', 'Event', 'TaggedRunMetadata', 'SessionLog', ] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py index bd46533572..12f120116f 100644 --- a/tensorflow/python/summary/writer/writer.py +++ b/tensorflow/python/summary/writer/writer.py @@ -25,6 +25,7 @@ from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import summary_pb2 from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.core.util import event_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.platform import gfile @@ -331,7 +332,20 @@ class FileWriter(SummaryToEventTransformer): graph_def: DEPRECATED: Use the `graph` argument instead. filename_suffix: A string. Every event file's name is suffixed with `suffix`. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + `FileWriter` is not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ + if context.in_eager_mode(): + raise RuntimeError( + "tf.summary.FileWriter is not compatible with eager execution. " + "Use tf.contrib.summary instead.") + event_writer = EventFileWriter(logdir, max_queue, flush_secs, filename_suffix) super(FileWriter, self).__init__(event_writer, graph, graph_def) -- GitLab From f17f389d88d2441302825e3afa5209fb3426002b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 14:35:06 -0700 Subject: [PATCH 530/573] Add a workaround in the Grappler arithmetic optimizer for the "Add" op not being marked commutative. This will allow Grappler to dedup nodes Add(x,y) and Add(y,x). PiperOrigin-RevId: 173950586 --- .../optimizers/arithmetic_optimizer.cc | 341 +++++++++--------- .../optimizers/arithmetic_optimizer_test.cc | 32 ++ 2 files changed, 207 insertions(+), 166 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 8ef3383aa3..400e1c017b 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -29,6 +29,180 @@ limitations under the License. namespace tensorflow { namespace grappler { +namespace { + +bool AreInversePermutations(gtl::ArraySlice a, + gtl::ArraySlice b) { + if (a.size() != b.size()) { + return false; + } + for (int i = 0; i < a.size(); ++i) { + if (a[b[i]] != i) { + return false; + } + } + return true; +} + +// Extract int32 values from a Const op to `int32_values`. Returns true if +// succeeds. +bool Int32ValuesFromNode(const NodeDef& node, std::vector* int32_values) { + if (node.op() != "Const") { + return false; + } + + if (node.attr().at("dtype").type() != DT_INT32) { + return false; + } + + // TensorProto represents the content of the tensor in either _val or + // tensor_content. + const TensorProto& tensor = node.attr().at("value").tensor(); + if (tensor.int_val_size() > 0 && tensor.has_tensor_shape()) { + // When tensor_shape is set, theoretically the representation of the data + // could be compressed. So, before copying int_val to the returned vector, + // make sure no compression happens. + const TensorShapeProto& shape = tensor.tensor_shape(); + if (shape.dim_size() == 1 && shape.dim(0).size() == tensor.int_val_size()) { + int32_values->insert(int32_values->end(), tensor.int_val().begin(), + tensor.int_val().end()); + } + return true; + } + + const auto tensor_content_size = tensor.tensor_content().size(); + if (tensor_content_size > 0) { + CHECK_EQ(0, tensor_content_size % sizeof(int32)) + << "tensor_content_size (" << tensor_content_size + << ") is not a multiple of " << sizeof(int32); + int32_values->resize(tensor_content_size / sizeof(int32)); + port::CopyToArray(tensor.tensor_content(), + reinterpret_cast(int32_values->data())); + return true; + } + + return false; +} + +bool SimplyReordersData(const NodeDef& node) { + return node.op() == "Transpose"; +} + +// Returns the data type in attribute `attr_name` of `node`. If that attribute +// doesn't exist, returns DT_INVALID. +DataType GetDataTypeFromAttr(const NodeDef& node, const string& attr_name) { + if (!node.attr().count(attr_name)) { + return DT_INVALID; + } + const auto& attr = node.attr().at(attr_name); + if (attr.value_case() != AttrValue::kType) { + return DT_INVALID; + } + return attr.type(); +} + +bool IsCommutative(const OpDef& op, const NodeDef& input1) { + if (op.name() == "Add") { + // Workaround for "Add" not being marked is_commutative and is_aggregate. + // (See cl/173915048). + const auto type = GetDataTypeFromAttr(input1, "T"); + return type != DT_INVALID && type != DT_STRING; + } + return op.is_commutative(); +} + +void SetDataTypeToAttr(DataType dtype, const string& attr_name, NodeDef* node) { + (*node->mutable_attr())[attr_name].set_type(dtype); +} + +string SourceDataTypeAttrName(const NodeDef& node) { + if (node.op() == "Bitcast") { + return "T"; + } else if (node.op() == "Cast") { + return "SrcT"; + } else { + LOG(FATAL) << "SourceDataTypeAttrName not implemented for op " << node.op(); + } +} + +string DestinationDataTypeAttrName(const NodeDef& node) { + if (node.op() == "Bitcast") { + return "type"; + } else if (node.op() == "Cast") { + return "DstT"; + } else { + LOG(FATAL) << "DestinationDataTypeAttrName not implemented for op " + << node.op(); + } +} + +DataType GetSourceDataType(const NodeDef& node) { + return GetDataTypeFromAttr(node, SourceDataTypeAttrName(node)); +} + +DataType GetDestinationDataType(const NodeDef& node) { + return GetDataTypeFromAttr(node, DestinationDataTypeAttrName(node)); +} + +void SetSourceDataType(DataType dtype, NodeDef* node) { + SetDataTypeToAttr(dtype, SourceDataTypeAttrName(*node), node); +} + +bool IsNumberType(DataType dtype) { + DataTypeVector number_types = NumberTypes(); + return std::find(number_types.begin(), number_types.end(), dtype) != + number_types.end(); +} + +const char kOutputShapesAttr[] = "_output_shapes"; + +// Returns whether `reshape` is an identity op. The tensor that `reshape` +// reshapes is the `output_pos`-th output of node `input`. +bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input, + const int output_pos) { + if (!reshape.attr().count(kOutputShapesAttr) || + !input.attr().count(kOutputShapesAttr)) { + return false; + } + + PartialTensorShape src_shape( + input.attr().at(kOutputShapesAttr).list().shape(output_pos)); + PartialTensorShape dst_shape( + reshape.attr().at(kOutputShapesAttr).list().shape(0)); + if (src_shape.unknown_rank() || dst_shape.unknown_rank()) { + return false; + } + + if (!dst_shape.IsCompatibleWith(src_shape)) { + return false; + } + + // Returns false when src_shape or dst_shape has >=2 dimensions with unknown + // sizes. + auto num_unknown_dim_sizes = [](const PartialTensorShape& partial_shape) { + auto dim_sizes = partial_shape.dim_sizes(); + return std::count(dim_sizes.begin(), dim_sizes.end(), -1); + }; + int src_num_unknown_dim_sizes = num_unknown_dim_sizes(src_shape); + int dst_num_unknown_dim_sizes = num_unknown_dim_sizes(dst_shape); + if (src_num_unknown_dim_sizes > 1 || dst_num_unknown_dim_sizes > 1) { + return false; + } + + // Now, src_shape and dst_shape have at most one dimension with unknown + // sizes, and are compatible. Therefore, the reshape is a no-op when + // + // 1. at least one of them is fully-defined, or + // 2. both are partially defined and the -1 appears on the same dimension, + // i.e., IsIdenticalTo returns true. + if (src_num_unknown_dim_sizes == 1 && dst_num_unknown_dim_sizes == 1) { + return dst_shape.IsIdenticalTo(src_shape); + } + + return true; +} + +} // namespace class UniqueNodes { public: @@ -86,7 +260,7 @@ bool UniqueNodes::SameNode(const NodeDef& node1, const NodeDef& node2) const { // Compare inputs. const OpDef* op_def = nullptr; Status status = OpRegistry::Global()->LookUpOpDef(node1.op(), &op_def); - const bool is_commutative = status.ok() && op_def->is_commutative(); + const bool is_commutative = status.ok() && IsCommutative(*op_def, node1); if (is_commutative) { std::vector inputs1(node1.input().begin(), node1.input().end()); std::vector inputs2(node2.input().begin(), node2.input().end()); @@ -102,7 +276,6 @@ bool UniqueNodes::SameNode(const NodeDef& node1, const NodeDef& node2) const { if (IsControlInput(node1.input(index))) { ctrl_inputs1.push_back(node1.input(index)); ctrl_inputs2.push_back(node2.input(index)); - } else { regular_inputs1.push_back(node1.input(index)); regular_inputs2.push_back(node2.input(index)); @@ -218,170 +391,6 @@ void ArithmeticOptimizer::DedupComputations(GraphDef* optimized_graph) const { } } -static bool AreInversePermutations(gtl::ArraySlice a, - gtl::ArraySlice b) { - if (a.size() != b.size()) { - return false; - } - for (int i = 0; i < a.size(); ++i) { - if (a[b[i]] != i) { - return false; - } - } - return true; -} - -// Extract int32 values from a Const op to `int32_values`. Returns true if -// succeeds. -static bool Int32ValuesFromNode(const NodeDef& node, - std::vector* int32_values) { - if (node.op() != "Const") { - return false; - } - - if (node.attr().at("dtype").type() != DT_INT32) { - return false; - } - - // TensorProto represents the content of the tensor in either _val or - // tensor_content. - const TensorProto& tensor = node.attr().at("value").tensor(); - if (tensor.int_val_size() > 0 && tensor.has_tensor_shape()) { - // When tensor_shape is set, theoretically the representation of the data - // could be compressed. So, before copying int_val to the returned vector, - // make sure no compression happens. - const TensorShapeProto& shape = tensor.tensor_shape(); - if (shape.dim_size() == 1 && shape.dim(0).size() == tensor.int_val_size()) { - int32_values->insert(int32_values->end(), tensor.int_val().begin(), - tensor.int_val().end()); - } - return true; - } - - const auto tensor_content_size = tensor.tensor_content().size(); - if (tensor_content_size > 0) { - CHECK_EQ(0, tensor_content_size % sizeof(int32)) - << "tensor_content_size (" << tensor_content_size - << ") is not a multiple of " << sizeof(int32); - int32_values->resize(tensor_content_size / sizeof(int32)); - port::CopyToArray(tensor.tensor_content(), - reinterpret_cast(int32_values->data())); - return true; - } - - return false; -} - -static bool SimplyReordersData(const NodeDef& node) { - return node.op() == "Transpose"; -} - -// Returns the data type in attribute `attr_name` of `node`. If that attribute -// doesn't exist, returns DT_INVALID. -static DataType GetDataTypeFromAttr(const NodeDef& node, - const string& attr_name) { - if (!node.attr().count(attr_name)) { - return DT_INVALID; - } - const auto& attr = node.attr().at(attr_name); - if (attr.value_case() != AttrValue::kType) { - return DT_INVALID; - } - return attr.type(); -} - -static void SetDataTypeToAttr(DataType dtype, const string& attr_name, - NodeDef* node) { - (*node->mutable_attr())[attr_name].set_type(dtype); -} - -static string SourceDataTypeAttrName(const NodeDef& node) { - if (node.op() == "Bitcast") { - return "T"; - } else if (node.op() == "Cast") { - return "SrcT"; - } else { - LOG(FATAL) << "SourceDataTypeAttrName not implemented for op " << node.op(); - } -} - -static string DestinationDataTypeAttrName(const NodeDef& node) { - if (node.op() == "Bitcast") { - return "type"; - } else if (node.op() == "Cast") { - return "DstT"; - } else { - LOG(FATAL) << "DestinationDataTypeAttrName not implemented for op " - << node.op(); - } -} - -static DataType GetSourceDataType(const NodeDef& node) { - return GetDataTypeFromAttr(node, SourceDataTypeAttrName(node)); -} - -static DataType GetDestinationDataType(const NodeDef& node) { - return GetDataTypeFromAttr(node, DestinationDataTypeAttrName(node)); -} - -static void SetSourceDataType(DataType dtype, NodeDef* node) { - SetDataTypeToAttr(dtype, SourceDataTypeAttrName(*node), node); -} - -static bool IsNumberType(DataType dtype) { - DataTypeVector number_types = NumberTypes(); - return std::find(number_types.begin(), number_types.end(), dtype) != - number_types.end(); -} - -const char kOutputShapesAttr[] = "_output_shapes"; - -// Returns whether `reshape` is an identity op. The tensor that `reshape` -// reshapes is the `output_pos`-th output of node `input`. -static bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input, - const int output_pos) { - if (!reshape.attr().count(kOutputShapesAttr) || - !input.attr().count(kOutputShapesAttr)) { - return false; - } - - PartialTensorShape src_shape( - input.attr().at(kOutputShapesAttr).list().shape(output_pos)); - PartialTensorShape dst_shape( - reshape.attr().at(kOutputShapesAttr).list().shape(0)); - if (src_shape.unknown_rank() || dst_shape.unknown_rank()) { - return false; - } - - if (!dst_shape.IsCompatibleWith(src_shape)) { - return false; - } - - // Returns false when src_shape or dst_shape has >=2 dimensions with unknown - // sizes. - auto num_unknown_dim_sizes = [](const PartialTensorShape& partial_shape) { - auto dim_sizes = partial_shape.dim_sizes(); - return std::count(dim_sizes.begin(), dim_sizes.end(), -1); - }; - int src_num_unknown_dim_sizes = num_unknown_dim_sizes(src_shape); - int dst_num_unknown_dim_sizes = num_unknown_dim_sizes(dst_shape); - if (src_num_unknown_dim_sizes > 1 || dst_num_unknown_dim_sizes > 1) { - return false; - } - - // Now, src_shape and dst_shape have at most one dimension with unknown - // sizes, and are compatible. Therefore, the reshape is a no-op when - // - // 1. at least one of them is fully-defined, or - // 2. both are partially defined and the -1 appears on the same dimension, - // i.e., IsIdenticalTo returns true. - if (src_num_unknown_dim_sizes == 1 && dst_num_unknown_dim_sizes == 1) { - return dst_shape.IsIdenticalTo(src_shape); - } - - return true; -} - string ArithmeticOptimizer::TrySimplifyAndReplaceUses( const NodeDef* node, GraphDef* graph_def, NodeMap* node_map, std::vector* new_nodes) const { diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc index a4de838a65..8edb34975f 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc @@ -77,6 +77,38 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) { EXPECT_EQ("c1", new_add.input(1)); } +TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output c1 = ops::Const(s.WithOpName("c1"), {1.0f, 2.0f}, {1, 2}); + Output c2 = ops::Const(s.WithOpName("c2"), {3.0f, 4.0f}, {1, 2}); + Output add1 = ops::Add(s.WithOpName("add1"), c1, c2); + Output add2 = ops::Add(s.WithOpName("add2"), c2, c1); + Output add3 = ops::Add(s.WithOpName("add3"), add1, add2); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ArithmeticOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + EXPECT_EQ(4, output.node_size()); + const NodeDef& new_c1 = output.node(0); + EXPECT_EQ("c1", new_c1.name()); + const NodeDef& new_c2 = output.node(1); + EXPECT_EQ("c2", new_c2.name()); + const NodeDef& new_add1 = output.node(2); + EXPECT_EQ("add1", new_add1.name()); + EXPECT_EQ(2, new_add1.input_size()); + EXPECT_EQ("c1", new_add1.input(0)); + EXPECT_EQ("c2", new_add1.input(1)); + const NodeDef& new_add3 = output.node(3); + EXPECT_EQ("add3", new_add3.name()); + EXPECT_EQ(2, new_add3.input_size()); + EXPECT_EQ("add1", new_add3.input(0)); + EXPECT_EQ("add1", new_add3.input(1)); +} + TEST_F(ArithmeticOptimizerTest, IdentityReshape) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output inputs = -- GitLab From a60cd87c4382ff18b45db3c41184b866fcdd7742 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 30 Oct 2017 15:00:41 -0700 Subject: [PATCH 531/573] No need for unique variable names in eager. PiperOrigin-RevId: 173954805 --- .../contrib/eager/python/metrics_test.py | 7 +- .../resource_variable_ops_test.py | 84 +++++++++---------- .../python/ops/resource_variable_ops.py | 10 +-- 3 files changed, 48 insertions(+), 53 deletions(-) diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 336ce9d307..2df596923b 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -121,9 +121,10 @@ class MetricsTest(test.TestCase): # accidentally share state. m1 = metrics.Mean() m1(0) - with self.assertRaises(ValueError): - m2 = metrics.Mean() - m2(2) + m2 = metrics.Mean() + m2(2) + self.assertAllEqual(0.0, m1.result()) + self.assertAllEqual(2.0, m2.result()) def testNamesWithSpaces(self): # Verify two metrics with the same class and name don't diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index c33bacc5a5..24ba1329f3 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -54,6 +54,18 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): 0, dtype=dtypes.int32)).run() + def testEagerNameNotIdentity(self): + with context.eager_mode(): + v0 = resource_variable_ops.ResourceVariable(1.0, name="a") + v1 = resource_variable_ops.ResourceVariable(2.0, name="a") + self.assertAllEqual(v0.numpy(), 1.0) + self.assertAllEqual(v1.numpy(), 2.0) + + def testEagerNameNotNeeded(self): + with context.eager_mode(): + v0 = resource_variable_ops.ResourceVariable(1.0) + self.assertAllEqual(v0.numpy(), 1.0) + def testReadVariableDtypeMismatchEager(self): with context.eager_mode(): handle = resource_variable_ops.var_handle_op( @@ -332,39 +344,38 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(ValueError, "No attr named '_class'"): _ = w.value().op.get_attr("_class") - @test_util.run_in_graph_and_eager_modes() def testSharedName(self): - v = resource_variable_ops.ResourceVariable(300.0, name="var4") - self.evaluate(variables.global_variables_initializer()) + with self.test_session(): + v = resource_variable_ops.ResourceVariable(300.0, name="var4") + variables.global_variables_initializer().run() - w = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4", - # Needed in Eager since we get a unique container name by default. - container=ops.get_default_graph()._container) - w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) - self.assertEqual(300.0, self.evaluate(w_read)) + w = resource_variable_ops.var_handle_op( + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4", + # Needed in Eager since we get a unique container name by default. + container=ops.get_default_graph()._container) + w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) + self.assertEqual(300.0, w_read.eval()) - x = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5", - container=ops.get_default_graph()._container) - with self.assertRaisesOpError("Resource .*/var5/.* does not exist"): - x_read = resource_variable_ops.read_variable_op(x, v.dtype.base_dtype) - self.evaluate(x_read) + x = resource_variable_ops.var_handle_op( + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5", + container=ops.get_default_graph()._container) + with self.assertRaisesOpError("Resource .*/var5/.* does not exist"): + resource_variable_ops.read_variable_op(x, v.dtype.base_dtype).eval() - @test_util.run_in_graph_and_eager_modes() def testSharedNameWithNamescope(self): - with ops.name_scope("foo"): - v = resource_variable_ops.ResourceVariable(300.0, name="var6") - self.assertEqual("foo/var6", v._shared_name) # pylint: disable=protected-access - self.assertEqual("foo/var6:0", v.name) - self.evaluate(variables.global_variables_initializer()) - - w = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6", - # Needed in Eager since we get a unique container name by default. - container=ops.get_default_graph()._container) - w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) - self.assertEqual(300.0, self.evaluate(w_read)) + with self.test_session(): + with ops.name_scope("foo"): + v = resource_variable_ops.ResourceVariable(300.0, name="var6") + self.assertEqual("foo/var6", v._shared_name) # pylint: disable=protected-access + self.assertEqual("foo/var6:0", v.name) + self.evaluate(variables.global_variables_initializer()) + + w = resource_variable_ops.var_handle_op( + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6", + # Needed in Eager since we get a unique container name by default. + container=ops.get_default_graph()._container) + w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) + self.assertEqual(300.0, self.evaluate(w_read)) @test_util.run_in_graph_and_eager_modes() def testShape(self): @@ -468,25 +479,10 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): name="var8") var.__del__() with self.assertRaisesRegexp(errors.NotFoundError, - r"Resource .*\/var8\/.* does not exist."): + r"Resource .* does not exist."): resource_variable_ops.destroy_resource_op(var._handle, ignore_lookup_error=False) - def testSharingViaResourceVariableObject(self): - with context.eager_mode(): - _ = resource_variable_ops.ResourceVariable(1.0, name="var0") - with self.assertRaisesRegexp(ValueError, - "'var0' already created"): - _ = resource_variable_ops.ResourceVariable(2.0, name="var0") - with ops.Graph().as_default(): - _ = resource_variable_ops.ResourceVariable(2.0, name="var0") - - def testVariableNameMissing(self): - with context.eager_mode(): - with self.assertRaisesRegexp(ValueError, - "Variables need to have explicit names"): - _ = resource_variable_ops.ResourceVariable(1.0) - if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index eebb5f217c..d7fb6767d1 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -43,6 +43,10 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): container = ops.get_default_graph()._container # pylint: disable=protected-access if container is None: container = "" + if not graph_mode: + # When in eager mode use a uid for the shared_name, to prevent accidental + # sharing. + shared_name = str(ops.uid()) handle = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype, shared_name=shared_name, name=name, @@ -293,12 +297,6 @@ class ResourceVariable(variables.Variable): # Save the graph's container prefix for error checking. Reading the value of # the ResourceVariable from another Graph in Eager mode is an error. self._container_prefix = ops.get_default_graph()._container_prefix # pylint: disable=protected-access - if not self._in_graph_mode and not name: - # TODO(ashankar,josh11b): make this unnecessary using the same - # logic as in layer - raise ValueError("Variables need to have explicit names when eager " - "execution is enabled") - with ops.control_dependencies(None): with ops.name_scope(name, "Variable", [] if init_from_fn else [initial_value]) as name: -- GitLab From 9aaa49a4e2cc41375fc7702a9bfb736a9c8ec92f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 15:10:19 -0700 Subject: [PATCH 532/573] Avoid using variables as booleans (similarly to tensors). PiperOrigin-RevId: 173956625 --- tensorflow/contrib/layers/python/layers/layers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index deeafdf300..c429d53cdc 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -1267,7 +1267,7 @@ def convolution2d_transpose( # Add variables to collections. _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias: + if layer.bias is not None: _add_variable_to_collections(layer.bias, variables_collections, 'biases') if normalizer_fn is not None: @@ -1376,7 +1376,7 @@ def convolution3d_transpose( # Add variables to collections. _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias: + if layer.bias is not None: _add_variable_to_collections(layer.bias, variables_collections, 'biases') if normalizer_fn is not None: @@ -2522,7 +2522,7 @@ def separable_convolution2d( variables_collections, 'weights') _add_variable_to_collections(layer.pointwise_kernel, variables_collections, 'weights') - if layer.bias: + if layer.bias is not None: _add_variable_to_collections(layer.bias, variables_collections, 'biases') -- GitLab From 8f7903b4c3699bc9129fa89d299699b1dfde6145 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 30 Oct 2017 15:43:18 -0700 Subject: [PATCH 533/573] Introduce SQLite SummaryWriterInterface This change allows tensors to be written from the graph, as they flow, directly to the database. Many of the important details haven't been implemented yet. This has been done with the new summary interface that's going to be used with eager. PiperOrigin-RevId: 173961448 --- tensorflow/contrib/tensorboard/db/BUILD | 28 ++ .../tensorboard/db/summary_db_writer.cc | 279 ++++++++++++++++++ .../tensorboard/db/summary_db_writer.h | 42 +++ .../tensorboard/db/summary_db_writer_test.cc | 162 ++++++++++ 4 files changed, 511 insertions(+) create mode 100644 tensorflow/contrib/tensorboard/db/summary_db_writer.cc create mode 100644 tensorflow/contrib/tensorboard/db/summary_db_writer.h create mode 100644 tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/contrib/tensorboard/db/BUILD index fb2d54916b..d8bbf87d2c 100644 --- a/tensorflow/contrib/tensorboard/db/BUILD +++ b/tensorflow/contrib/tensorboard/db/BUILD @@ -22,8 +22,36 @@ tf_cc_test( srcs = ["schema_test.cc"], deps = [ ":schema", + "//tensorflow/core:lib", "//tensorflow/core:test", "//tensorflow/core:test_main", + "//tensorflow/core/lib/db:sqlite", + ], +) + +cc_library( + name = "summary_db_writer", + srcs = ["summary_db_writer.cc"], + hdrs = ["summary_db_writer.h"], + deps = [ + ":schema", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/kernels:summary_interface", + "//tensorflow/core/lib/db:sqlite", + ], +) + +tf_cc_test( + name = "summary_db_writer_test", + srcs = ["summary_db_writer_test.cc"], + deps = [ + ":summary_db_writer", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/db:sqlite", ], ) diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.cc b/tensorflow/contrib/tensorboard/db/summary_db_writer.cc new file mode 100644 index 0000000000..df64e36305 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/summary_db_writer.cc @@ -0,0 +1,279 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h" + +#include "tensorflow/contrib/tensorboard/db/schema.h" +#include "tensorflow/core/lib/db/sqlite.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/lib/strings/stringprintf.h" +#include "tensorflow/core/platform/snappy.h" + +namespace tensorflow { +namespace { + +int64 MakeRandomId() { + int64 id = static_cast(random::New64() & ((1ULL << 63) - 1)); + if (id == 0) { + ++id; + } + return id; +} + +class SummaryDbWriter : public SummaryWriterInterface { + public: + SummaryDbWriter(Env* env, std::shared_ptr db) + : SummaryWriterInterface(), env_(env), db_(std::move(db)), run_id_(-1) {} + ~SummaryDbWriter() override {} + + Status Initialize(const string& experiment_name, const string& run_name, + const string& user_name) { + mutex_lock ml(mu_); + insert_tensor_ = db_->Prepare(R"sql( + INSERT OR REPLACE INTO Tensors (tag_id, step, computed_time, tensor) + VALUES (?, ?, ?, ?) + )sql"); + update_metadata_ = db_->Prepare(R"sql( + UPDATE Tags SET metadata = ? WHERE tag_id = ? + )sql"); + experiment_name_ = experiment_name; + run_name_ = run_name; + user_name_ = user_name; + return Status::OK(); + } + + // TODO(@jart): Use transactions that COMMIT on Flush() + // TODO(@jart): Retry Commit() on SQLITE_BUSY with exponential back-off. + Status Flush() override { return Status::OK(); } + + Status WriteTensor(int64 global_step, Tensor t, const string& tag, + const string& serialized_metadata) override { + mutex_lock ml(mu_); + TF_RETURN_IF_ERROR(InitializeParents()); + // TODO(@jart): Memoize tag_id. + int64 tag_id; + TF_RETURN_IF_ERROR(GetTagId(run_id_, tag, &tag_id)); + if (!serialized_metadata.empty()) { + // TODO(@jart): Only update metadata for first tensor. + update_metadata_.BindBlobUnsafe(1, serialized_metadata); + update_metadata_.BindInt(2, tag_id); + TF_RETURN_IF_ERROR(update_metadata_.StepAndReset()); + } + // TODO(@jart): Lease blocks of rowids and *_ids to minimize fragmentation. + // TODO(@jart): Check for random ID collisions without needing txn retry. + insert_tensor_.BindInt(1, tag_id); + insert_tensor_.BindInt(2, global_step); + insert_tensor_.BindDouble(3, GetWallTime()); + switch (t.dtype()) { + case DT_INT64: + insert_tensor_.BindInt(4, t.scalar()()); + break; + case DT_DOUBLE: + insert_tensor_.BindDouble(4, t.scalar()()); + break; + default: + TF_RETURN_IF_ERROR(BindTensor(t)); + break; + } + TF_RETURN_IF_ERROR(insert_tensor_.StepAndReset()); + return Status::OK(); + } + + Status WriteEvent(std::unique_ptr e) override { + // TODO(@jart): This will be used to load event logs. + return errors::Unimplemented("WriteEvent"); + } + + Status WriteScalar(int64 global_step, Tensor t, const string& tag) override { + // TODO(@jart): Unlike WriteTensor, this method would be granted leniency + // to change the dtype if it saves storage space. For example, + // DT_UINT32 would be stored in the database as an INTEGER + // rather than a serialized BLOB. But when reading it back, + // the dtype would become DT_INT64. + return errors::Unimplemented("WriteScalar"); + } + + Status WriteHistogram(int64 global_step, Tensor t, + const string& tag) override { + return errors::Unimplemented( + "SummaryDbWriter::WriteHistogram not supported. Please use ", + "tensorboard.summary.histogram() instead."); + } + + Status WriteImage(int64 global_step, Tensor tensor, const string& tag, + int max_images, Tensor bad_color) override { + return errors::Unimplemented( + "SummaryDbWriter::WriteImage not supported. Please use ", + "tensorboard.summary.image() instead."); + } + + Status WriteAudio(int64 global_step, Tensor tensor, const string& tag, + int max_outputs, float sample_rate) override { + return errors::Unimplemented( + "SummaryDbWriter::WriteAudio not supported. Please use ", + "tensorboard.summary.audio() instead."); + } + + string DebugString() override { return "SummaryDbWriter"; } + + private: + double GetWallTime() { + // TODO(@jart): Follow precise definitions for time laid out in schema. + // TODO(@jart): Use monotonic clock from gRPC codebase. + return static_cast(env_->NowMicros()) / 1.0e6; + } + + Status BindTensor(const Tensor& t) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // TODO(@jart): Make portable between little and big endian systems. + // TODO(@jart): Use TensorChunks with minimal copying for big tensors. + TensorProto p; + t.AsProtoTensorContent(&p); + string encoded; + if (!p.SerializeToString(&encoded)) { + return errors::DataLoss("SerializeToString failed"); + } + // TODO(@jart): Put byte at beginning of blob to indicate encoding. + // TODO(@jart): Allow crunch tool to re-compress with zlib instead. + string compressed; + if (!port::Snappy_Compress(encoded.data(), encoded.size(), &compressed)) { + return errors::FailedPrecondition("TensorBase needs Snappy"); + } + insert_tensor_.BindBlobUnsafe(4, compressed); + return Status::OK(); + } + + Status InitializeParents() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (run_id_ >= 0) { + return Status::OK(); + } + int64 user_id; + TF_RETURN_IF_ERROR(GetUserId(user_name_, &user_id)); + int64 experiment_id; + TF_RETURN_IF_ERROR( + GetExperimentId(user_id, experiment_name_, &experiment_id)); + TF_RETURN_IF_ERROR(GetRunId(experiment_id, run_name_, &run_id_)); + return Status::OK(); + } + + Status GetUserId(const string& user_name, int64* user_id) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (user_name.empty()) { + *user_id = 0LL; + return Status::OK(); + } + SqliteStatement get_user_id = db_->Prepare(R"sql( + SELECT user_id FROM Users WHERE user_name = ? + )sql"); + get_user_id.BindText(1, user_name); + bool is_done; + TF_RETURN_IF_ERROR(get_user_id.Step(&is_done)); + if (!is_done) { + *user_id = get_user_id.ColumnInt(0); + } else { + *user_id = MakeRandomId(); + SqliteStatement insert_user = db_->Prepare(R"sql( + INSERT INTO Users (user_id, user_name, inserted_time) VALUES (?, ?, ?) + )sql"); + insert_user.BindInt(1, *user_id); + insert_user.BindText(2, user_name); + insert_user.BindDouble(3, GetWallTime()); + TF_RETURN_IF_ERROR(insert_user.StepAndReset()); + } + return Status::OK(); + } + + Status GetExperimentId(int64 user_id, const string& experiment_name, + int64* experiment_id) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // TODO(@jart): Compute started_time. + return GetId("Experiments", "user_id", user_id, "experiment_name", + experiment_name, "experiment_id", experiment_id); + } + + Status GetRunId(int64 experiment_id, const string& run_name, int64* run_id) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // TODO(@jart): Compute started_time. + return GetId("Runs", "experiment_id", experiment_id, "run_name", run_name, + "run_id", run_id); + } + + Status GetTagId(int64 run_id, const string& tag_name, int64* tag_id) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return GetId("Tags", "run_id", run_id, "tag_name", tag_name, "tag_id", + tag_id); + } + + Status GetId(const char* table, const char* parent_id_field, int64 parent_id, + const char* name_field, const string& name, const char* id_field, + int64* id) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (name.empty()) { + *id = 0LL; + return Status::OK(); + } + SqliteStatement select = db_->Prepare( + strings::Printf("SELECT %s FROM %s WHERE %s = ? AND %s = ?", id_field, + table, parent_id_field, name_field)); + if (parent_id > 0) { + select.BindInt(1, parent_id); + } + select.BindText(2, name); + bool is_done; + TF_RETURN_IF_ERROR(select.Step(&is_done)); + if (!is_done) { + *id = select.ColumnInt(0); + } else { + *id = MakeRandomId(); + SqliteStatement insert = db_->Prepare(strings::Printf( + "INSERT INTO %s (%s, %s, %s, inserted_time) VALUES (?, ?, ?, ?)", + table, parent_id_field, id_field, name_field)); + if (parent_id > 0) { + insert.BindInt(1, parent_id); + } + insert.BindInt(2, *id); + insert.BindText(3, name); + insert.BindDouble(4, GetWallTime()); + TF_RETURN_IF_ERROR(insert.StepAndReset()); + } + return Status::OK(); + } + + mutex mu_; + Env* env_; + std::shared_ptr db_ GUARDED_BY(mu_); + SqliteStatement insert_tensor_ GUARDED_BY(mu_); + SqliteStatement update_metadata_ GUARDED_BY(mu_); + string user_name_ GUARDED_BY(mu_); + string experiment_name_ GUARDED_BY(mu_); + string run_name_ GUARDED_BY(mu_); + int64 run_id_ GUARDED_BY(mu_); +}; + +} // namespace + +Status CreateSummaryDbWriter(std::shared_ptr db, + const string& experiment_name, + const string& run_name, const string& user_name, + Env* env, SummaryWriterInterface** result) { + TF_RETURN_IF_ERROR(SetupTensorboardSqliteDb(db)); + SummaryDbWriter* w = new SummaryDbWriter(env, std::move(db)); + const Status s = w->Initialize(experiment_name, run_name, user_name); + if (!s.ok()) { + w->Unref(); + *result = nullptr; + return s; + } + *result = w; + return Status::OK(); +} + +} // namespace tensorflow diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.h b/tensorflow/contrib/tensorboard/db/summary_db_writer.h new file mode 100644 index 0000000000..74f61e50b7 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/summary_db_writer.h @@ -0,0 +1,42 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_ +#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_ + +#include "tensorflow/core/kernels/summary_interface.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/db/sqlite.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +/// \brief Creates SQLite SummaryWriterInterface. +/// +/// This can be used to write tensors from the execution graph directly +/// to a database. The schema will be created automatically, but only +/// if necessary. Entries in the Users, Experiments, and Runs tables +/// will be created automatically if they don't already exist. +/// +/// Please note that the type signature of this function may change in +/// the future if support for other DBs is added to core. +Status CreateSummaryDbWriter(std::shared_ptr db, + const string& experiment_name, + const string& run_name, const string& user_name, + Env* env, SummaryWriterInterface** result); + +} // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_ diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc b/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc new file mode 100644 index 0000000000..d32904f97c --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc @@ -0,0 +1,162 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h" + +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/db/sqlite.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +Tensor MakeScalarInt64(int64 x) { + Tensor t(DT_INT64, TensorShape({})); + t.scalar()() = x; + return t; +} + +class FakeClockEnv : public EnvWrapper { + public: + FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {} + void AdvanceByMillis(const uint64 millis) { current_millis_ += millis; } + uint64 NowMicros() override { return current_millis_ * 1000; } + uint64 NowSeconds() override { return current_millis_ * 1000; } + + private: + uint64 current_millis_; +}; + +class SummaryDbWriterTest : public ::testing::Test { + protected: + void SetUp() override { db_ = Sqlite::Open("file::memory:").ValueOrDie(); } + + void TearDown() override { + if (writer_ != nullptr) { + writer_->Unref(); + writer_ = nullptr; + } + } + + int64 QueryInt(const string& sql) { + SqliteStatement stmt = db_->Prepare(sql); + bool is_done; + Status s = stmt.Step(&is_done); + if (!s.ok() || is_done) { + LOG(ERROR) << s << " due to " << sql; + return -1; + } + return stmt.ColumnInt(0); + } + + double QueryDouble(const string& sql) { + SqliteStatement stmt = db_->Prepare(sql); + bool is_done; + Status s = stmt.Step(&is_done); + if (!s.ok() || is_done) { + LOG(ERROR) << s << " due to " << sql; + return -1; + } + return stmt.ColumnDouble(0); + } + + string QueryString(const string& sql) { + SqliteStatement stmt = db_->Prepare(sql); + bool is_done; + Status s = stmt.Step(&is_done); + if (!s.ok() || is_done) { + LOG(ERROR) << s << " due to " << sql; + return "MISSINGNO"; + } + return stmt.ColumnString(0); + } + + FakeClockEnv env_; + std::shared_ptr db_; + SummaryWriterInterface* writer_ = nullptr; +}; + +TEST_F(SummaryDbWriterTest, NothingWritten_NoRowsCreated) { + TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_, + &writer_)); + TF_ASSERT_OK(writer_->Flush()); + writer_->Unref(); + writer_ = nullptr; + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Users")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Experiments")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Runs")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Tags")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Tensors")); +} + +TEST_F(SummaryDbWriterTest, TensorsWritten_RowsGetInitialized) { + TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_, + &writer_)); + env_.AdvanceByMillis(23); + TF_ASSERT_OK(writer_->WriteTensor(1, MakeScalarInt64(123LL), "taggy", + "this-is-metaaa")); + env_.AdvanceByMillis(23); + TF_ASSERT_OK(writer_->WriteTensor(2, MakeScalarInt64(314LL), "taggy", "")); + TF_ASSERT_OK(writer_->Flush()); + + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Users")); + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Experiments")); + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Runs")); + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Tags")); + ASSERT_EQ(2LL, QueryInt("SELECT COUNT(*) FROM Tensors")); + + int64 user_id = QueryInt("SELECT user_id FROM Users"); + int64 experiment_id = QueryInt("SELECT experiment_id FROM Experiments"); + int64 run_id = QueryInt("SELECT run_id FROM Runs"); + int64 tag_id = QueryInt("SELECT tag_id FROM Tags"); + EXPECT_LT(0LL, user_id); + EXPECT_LT(0LL, experiment_id); + EXPECT_LT(0LL, run_id); + EXPECT_LT(0LL, tag_id); + + EXPECT_EQ("jart", QueryString("SELECT user_name FROM Users")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Users")); + + EXPECT_EQ(user_id, QueryInt("SELECT user_id FROM Experiments")); + EXPECT_EQ("mad-science", + QueryString("SELECT experiment_name FROM Experiments")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Experiments")); + + EXPECT_EQ(experiment_id, QueryInt("SELECT experiment_id FROM Runs")); + EXPECT_EQ("train", QueryString("SELECT run_name FROM Runs")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Runs")); + + EXPECT_EQ(run_id, QueryInt("SELECT run_id FROM Tags")); + EXPECT_EQ("taggy", QueryString("SELECT tag_name FROM Tags")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Tags")); + EXPECT_EQ("this-is-metaaa", QueryString("SELECT metadata FROM Tags")); + + EXPECT_EQ(tag_id, QueryInt("SELECT tag_id FROM Tensors WHERE step = 1")); + EXPECT_EQ(0.023, + QueryDouble("SELECT computed_time FROM Tensors WHERE step = 1")); + EXPECT_EQ("this-is-metaaa", QueryString("SELECT metadata FROM Tags")); + EXPECT_FALSE( + QueryString("SELECT tensor FROM Tensors WHERE step = 1").empty()); + + EXPECT_EQ(tag_id, QueryInt("SELECT tag_id FROM Tensors WHERE step = 2")); + EXPECT_EQ(0.046, + QueryDouble("SELECT computed_time FROM Tensors WHERE step = 2")); + EXPECT_EQ("this-is-metaaa", QueryString("SELECT metadata FROM Tags")); + EXPECT_FALSE( + QueryString("SELECT tensor FROM Tensors WHERE step = 2").empty()); +} + +} // namespace +} // namespace tensorflow -- GitLab From 89120eb688008f9fbf0cbbc5f1984abd90577d63 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 30 Oct 2017 15:58:31 -0700 Subject: [PATCH 534/573] scatter_update for resource variables PiperOrigin-RevId: 173963715 --- .../core/kernels/resource_variable_ops.cc | 8 +-- tensorflow/core/ops/resource_variable_ops.cc | 42 ++++++++++++++- .../resource_variable_ops_test.py | 6 +++ tensorflow/python/ops/state_ops.py | 52 +++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index a4db4abd7b..217fb3b781 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -569,9 +569,11 @@ class ResourceScatterUpdateOp : public OpKernel { REGISTER_SCATTER_KERNEL_INDEX(type, int64, dev, name, op); // TODO(apassos) add the other types here. -#define REGISTER_SCATTER_ARITHEMTIC(type, dev) \ - REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterAdd", \ - scatter_op::UpdateOp::ADD); +#define REGISTER_SCATTER_ARITHEMTIC(type, dev) \ + REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterAdd", \ + scatter_op::UpdateOp::ADD); \ + REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterUpdate", \ + scatter_op::UpdateOp::ASSIGN); // Registers CPU kernels. #define REGISTER_SCATTER_ARITHEMTIC_CPU(type) \ diff --git a/tensorflow/core/ops/resource_variable_ops.cc b/tensorflow/core/ops/resource_variable_ops.cc index c4802a1cc1..cdfbec85cf 100644 --- a/tensorflow/core/ops/resource_variable_ops.cc +++ b/tensorflow/core/ops/resource_variable_ops.cc @@ -311,7 +311,7 @@ the same location, their contributions add. Requires `updates.shape = indices.shape + ref.shape[1:]`.
- +
resource: Should be from a `Variable` node. @@ -319,4 +319,44 @@ indices: A tensor of indices into the first dimension of `ref`. updates: A tensor of updated values to add to `ref`. )doc"); +REGISTER_OP("ResourceScatterUpdate") + .Input("resource: resource") + .Input("indices: Tindices") + .Input("updates: dtype") + .Attr("dtype: numbertype") + .Attr("Tindices: {int32, int64}") + .SetShapeFn([](InferenceContext* c) { + ShapeAndType handle_shape_and_type; + TF_RETURN_IF_ERROR( + ValidateVariableResourceHandle(c, &handle_shape_and_type)); + ShapeHandle var_shape = handle_shape_and_type.shape; + ShapeHandle indices_shape = c->input(1); + + ShapeHandle unused_updates_shape; + ShapeHandle concat; + ShapeHandle var_subshape; + TF_RETURN_IF_ERROR(c->Subshape(var_shape, 1, &var_subshape)); + TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, var_subshape, &concat)); + TF_RETURN_IF_ERROR(c->Merge(c->input(2), concat, &unused_updates_shape)); + return Status::OK(); + }) + .Doc(R"doc( +Assigns sparse updates to the variable referenced by `resource`. + +This operation computes + + # Scalar indices + ref[indices, ...] = updates[...] + + # Vector indices (for each i) + ref[indices[i], ...] = updates[i, ...] + + # High rank indices (for each i, ..., j) + ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] + +resource: Should be from a `Variable` node. +indices: A tensor of indices into the first dimension of `ref`. +updates: A tensor of updated values to add to `ref`. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 24ba1329f3..7922e3838f 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -483,6 +483,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): resource_variable_ops.destroy_resource_op(var._handle, ignore_lookup_error=False) + def testScatterUpdate(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable([1.0, 2.0], name="update") + state_ops.scatter_update(v, [1], [3.0]) + self.assertAllEqual([1.0, 3.0], v.numpy()) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py index 5b9ca7c0b9..dbab07da42 100644 --- a/tensorflow/python/ops/state_ops.py +++ b/tensorflow/python/ops/state_ops.py @@ -297,3 +297,55 @@ def count_up_to(ref, limit, name=None): return gen_state_ops.count_up_to(ref, limit=limit, name=name) return gen_state_ops.resource_count_up_to( ref.handle, limit, T=ref.dtype, name=name) + + +def scatter_update(ref, indices, updates, use_locking=True, name=None): + # pylint: disable=line-too-long + r"""Applies sparse updates to a variable reference. + + This operation computes + + ```python + # Scalar indices + ref[indices, ...] = updates[...] + + # Vector indices (for each i) + ref[indices[i], ...] = updates[i, ...] + + # High rank indices (for each i, ..., j) + ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] + ``` + + This operation outputs `ref` after the update is done. + This makes it easier to chain operations that need to use the reset value. + + If values in `ref` is to be updated more than once, because there are + duplicate entries in `indices`, the order at which the updates happen + for each value is undefined. + + Requires `updates.shape = indices.shape + ref.shape[1:]`. + +
+ +
+ + Args: + ref: A `Variable`. + indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. + A tensor of indices into the first dimension of `ref`. + updates: A `Tensor`. Must have the same type as `ref`. + A tensor of updated values to store in `ref`. + use_locking: An optional `bool`. Defaults to `True`. + If True, the assignment will be protected by a lock; + otherwise the behavior is undefined, but may exhibit less contention. + name: A name for the operation (optional). + + Returns: + Same as `ref`. Returned as a convenience for operations that want + to use the updated values after the update is done. + """ + if ref.dtype._is_ref_dtype: + return gen_state_ops.scatter_update(ref, indices, updates, + use_locking=use_locking, name=name) + return gen_resource_variable_ops.resource_scatter_update( + ref.handle, indices, updates, name=name) -- GitLab From 302ab0ff761600083091a07e3f167be7896b47d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:08:09 -0700 Subject: [PATCH 535/573] Update ops-related pbtxt files. PiperOrigin-RevId: 173965174 --- .../core/ops/compat/ops_history.v1.pbtxt | 50 ++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 57 ++++++++++++++++++- 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 35df6e89fa..f385ef54f1 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -27689,6 +27689,56 @@ op { } is_stateful: true } +op { + name: "ResourceScatterUpdate" + input_arg { + name: "resource" + type: DT_RESOURCE + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "dtype" + } + attr { + name: "dtype" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + is_stateful: true +} op { name: "ResourceSparseApplyAdadelta" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 9f255f13c4..4017a46521 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -22337,7 +22337,62 @@ op { } } summary: "Adds sparse updates to the variable referenced by `resource`." - description: "This operation computes\n\n # Scalar indices\n ref[indices, ...] += updates[...]\n\n # Vector indices (for each i)\n ref[indices[i], ...] += updates[i, ...]\n\n # High rank indices (for each i, ..., j)\n ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]\n\nDuplicate entries are handled correctly: if multiple `indices` reference\nthe same location, their contributions add.\n\nRequires `updates.shape = indices.shape + ref.shape[1:]`.\n\n
\n\n
" + description: "This operation computes\n\n # Scalar indices\n ref[indices, ...] += updates[...]\n\n # Vector indices (for each i)\n ref[indices[i], ...] += updates[i, ...]\n\n # High rank indices (for each i, ..., j)\n ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]\n\nDuplicate entries are handled correctly: if multiple `indices` reference\nthe same location, their contributions add.\n\nRequires `updates.shape = indices.shape + ref.shape[1:]`.\n\n
\n\n
" + is_stateful: true +} +op { + name: "ResourceScatterUpdate" + input_arg { + name: "resource" + description: "Should be from a `Variable` node." + type: DT_RESOURCE + } + input_arg { + name: "indices" + description: "A tensor of indices into the first dimension of `ref`." + type_attr: "Tindices" + } + input_arg { + name: "updates" + description: "A tensor of updated values to add to `ref`." + type_attr: "dtype" + } + attr { + name: "dtype" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + summary: "Assigns sparse updates to the variable referenced by `resource`." + description: "This operation computes\n\n # Scalar indices\n ref[indices, ...] = updates[...]\n\n # Vector indices (for each i)\n ref[indices[i], ...] = updates[i, ...]\n\n # High rank indices (for each i, ..., j)\n ref[indices[i, ..., j], ...] = updates[i, ..., j, ...]" is_stateful: true } op { -- GitLab From f9a673cb71da60323343fa62b76a2577466b0aa7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:09:32 -0700 Subject: [PATCH 536/573] In the overloaded HloVerifier::CheckShape, include the failing instruction in the error message. PiperOrigin-RevId: 173965368 --- tensorflow/compiler/xla/service/hlo_verifier.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index f3a098057b..86ae00971b 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -359,7 +359,10 @@ class ShapeVerifier : public DfsHloVisitor { Status CheckShape(const HloInstruction* instruction, const StatusOr& expected_shape_status) { if (!expected_shape_status.ok()) { - return expected_shape_status.status(); + Status s = expected_shape_status.status(); + tensorflow::errors::AppendToMessage(&s, ", for instruction ", + instruction->ToString()); + return s; } return CheckShape(instruction, expected_shape_status.ValueOrDie()); } -- GitLab From 558f146e1d84a6dbca5282dfeefdbd6312eb97ba Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:14:34 -0700 Subject: [PATCH 537/573] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 173966068 --- tensorflow/go/op/wrappers.go | 1154 +++++++++++++++++----------------- 1 file changed, 593 insertions(+), 561 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 2f8a06a632..f316096963 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -7996,146 +7996,6 @@ func Cholesky(scope *Scope, input tf.Output) (output tf.Output) { return op.Output(0) } -// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad. -type FusedBatchNormGradAttr func(optionalAttr) - -// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value. -// -// value: A small float number added to the variance of x. -// If not specified, defaults to 0.0001 -func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr { - return func(m optionalAttr) { - m["epsilon"] = value - } -} - -// FusedBatchNormGradDataFormat sets the optional data_format attribute to value. -// -// value: The data format for y_backprop, x, x_backprop. -// Either "NHWC" (default) or "NCHW". -// If not specified, defaults to "NHWC" -func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr { - return func(m optionalAttr) { - m["data_format"] = value - } -} - -// FusedBatchNormGradIsTraining sets the optional is_training attribute to value. -// -// value: A bool value to indicate the operation is for training (default) -// or inference. -// If not specified, defaults to true -func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr { - return func(m optionalAttr) { - m["is_training"] = value - } -} - -// Gradient for batch normalization. -// -// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW". -// The size of 1D Tensors matches the dimension C of the 4D Tensors. -// -// Arguments: -// y_backprop: A 4D Tensor for the gradient with respect to y. -// x: A 4D Tensor for input data. -// scale: A 1D Tensor for scaling factor, to scale the normalized x. -// reserve_space_1: When is_training is True, a 1D Tensor for the computed batch -// mean to be reused in gradient computation. When is_training is -// False, a 1D Tensor for the population mean to be reused in both -// 1st and 2nd order gradient computation. -// reserve_space_2: When is_training is True, a 1D Tensor for the computed batch -// variance (inverted variance in the cuDNN case) to be reused in -// gradient computation. When is_training is False, a 1D Tensor -// for the population variance to be reused in both 1st and 2nd -// order gradient computation. -// -// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input -// in FusedBatchNorm. -func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradAttr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "FusedBatchNormGrad", - Input: []tf.Input{ - y_backprop, x, scale, reserve_space_1, reserve_space_2, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4) -} - -// ShapeAttr is an optional argument to Shape. -type ShapeAttr func(optionalAttr) - -// ShapeOutType sets the optional out_type attribute to value. -// If not specified, defaults to DT_INT32 -func ShapeOutType(value tf.DataType) ShapeAttr { - return func(m optionalAttr) { - m["out_type"] = value - } -} - -// Returns the shape of a tensor. -// -// This operation returns a 1-D integer tensor representing the shape of `input`. -// -// For example: -// -// ``` -// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]] -// shape(t) ==> [2, 2, 3] -// ``` -func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "Shape", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Computes softmax cross entropy cost and gradients to backpropagate. -// -// Inputs are the logits, not probabilities. -// -// Arguments: -// features: batch_size x num_classes matrix -// labels: batch_size x num_classes matrix -// The caller must ensure that each batch of labels represents a valid -// probability distribution. -// -// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix). -func SoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SoftmaxCrossEntropyWithLogits", - Input: []tf.Input{ - features, labels, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - // MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad. type MaxPool3DGradGradAttr func(optionalAttr) @@ -10312,7 +10172,7 @@ func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples t // Requires `updates.shape = indices.shape + ref.shape[1:]`. // //
-// +// //
// // Arguments: @@ -13575,45 +13435,6 @@ func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, pa return op.Output(0) } -// L2 Loss. -// -// Computes half the L2 norm of a tensor without the `sqrt`: -// -// output = sum(t ** 2) / 2 -// -// Arguments: -// t: Typically 2-D, but may have any dimensions. -// -// Returns 0-D. -func L2Loss(scope *Scope, t tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "L2Loss", - Input: []tf.Input{ - t, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Computes rectified linear: `max(features, 0)`. -func Relu(scope *Scope, features tf.Output) (activations tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Relu", - Input: []tf.Input{ - features, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Returns the truth value of (x >= y) element-wise. // // *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting @@ -14684,57 +14505,236 @@ func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) { return op.Output(0) } -// Get the value of the tensor specified by its handle. -// -// Arguments: -// handle: The handle for a tensor stored in the session state. -// dtype: The type of the output value. +// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad. +type FusedBatchNormGradAttr func(optionalAttr) + +// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value. // -// Returns The tensor for the given handle. -func GetSessionTensor(scope *Scope, handle tf.Output, dtype tf.DataType) (value tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtype": dtype} - opspec := tf.OpSpec{ - Type: "GetSessionTensor", - Input: []tf.Input{ - handle, - }, - Attrs: attrs, +// value: A small float number added to the variance of x. +// If not specified, defaults to 0.0001 +func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr { + return func(m optionalAttr) { + m["epsilon"] = value } - op := scope.AddOperation(opspec) - return op.Output(0) } -// ResourceApplyProximalGradientDescentAttr is an optional argument to ResourceApplyProximalGradientDescent. -type ResourceApplyProximalGradientDescentAttr func(optionalAttr) +// FusedBatchNormGradDataFormat sets the optional data_format attribute to value. +// +// value: The data format for y_backprop, x, x_backprop. +// Either "NHWC" (default) or "NCHW". +// If not specified, defaults to "NHWC" +func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr { + return func(m optionalAttr) { + m["data_format"] = value + } +} -// ResourceApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value. +// FusedBatchNormGradIsTraining sets the optional is_training attribute to value. // -// value: If True, the subtraction will be protected by a lock; -// otherwise the behavior is undefined, but may exhibit less contention. -// If not specified, defaults to false -func ResourceApplyProximalGradientDescentUseLocking(value bool) ResourceApplyProximalGradientDescentAttr { +// value: A bool value to indicate the operation is for training (default) +// or inference. +// If not specified, defaults to true +func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr { return func(m optionalAttr) { - m["use_locking"] = value + m["is_training"] = value } } -// Update '*var' as FOBOS algorithm with fixed learning rate. +// Gradient for batch normalization. // -// prox_v = var - alpha * delta -// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} +// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW". +// The size of 1D Tensors matches the dimension C of the 4D Tensors. // // Arguments: -// var_: Should be from a Variable(). -// alpha: Scaling factor. Must be a scalar. -// l1: L1 regularization. Must be a scalar. -// l2: L2 regularization. Must be a scalar. -// delta: The change. +// y_backprop: A 4D Tensor for the gradient with respect to y. +// x: A 4D Tensor for input data. +// scale: A 1D Tensor for scaling factor, to scale the normalized x. +// reserve_space_1: When is_training is True, a 1D Tensor for the computed batch +// mean to be reused in gradient computation. When is_training is +// False, a 1D Tensor for the population mean to be reused in both +// 1st and 2nd order gradient computation. +// reserve_space_2: When is_training is True, a 1D Tensor for the computed batch +// variance (inverted variance in the cuDNN case) to be reused in +// gradient computation. When is_training is False, a 1D Tensor +// for the population variance to be reused in both 1st and 2nd +// order gradient computation. // -// Returns the created operation. -func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, delta tf.Output, optional ...ResourceApplyProximalGradientDescentAttr) (o *tf.Operation) { +// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input +// in FusedBatchNorm. +func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradAttr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "FusedBatchNormGrad", + Input: []tf.Input{ + y_backprop, x, scale, reserve_space_1, reserve_space_2, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4) +} + +// Computes rectified linear: `max(features, 0)`. +func Relu(scope *Scope, features tf.Output) (activations tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Relu", + Input: []tf.Input{ + features, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// L2 Loss. +// +// Computes half the L2 norm of a tensor without the `sqrt`: +// +// output = sum(t ** 2) / 2 +// +// Arguments: +// t: Typically 2-D, but may have any dimensions. +// +// Returns 0-D. +func L2Loss(scope *Scope, t tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "L2Loss", + Input: []tf.Input{ + t, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ShapeAttr is an optional argument to Shape. +type ShapeAttr func(optionalAttr) + +// ShapeOutType sets the optional out_type attribute to value. +// If not specified, defaults to DT_INT32 +func ShapeOutType(value tf.DataType) ShapeAttr { + return func(m optionalAttr) { + m["out_type"] = value + } +} + +// Returns the shape of a tensor. +// +// This operation returns a 1-D integer tensor representing the shape of `input`. +// +// For example: +// +// ``` +// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]] +// shape(t) ==> [2, 2, 3] +// ``` +func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Shape", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes softmax cross entropy cost and gradients to backpropagate. +// +// Inputs are the logits, not probabilities. +// +// Arguments: +// features: batch_size x num_classes matrix +// labels: batch_size x num_classes matrix +// The caller must ensure that each batch of labels represents a valid +// probability distribution. +// +// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix). +func SoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SoftmaxCrossEntropyWithLogits", + Input: []tf.Input{ + features, labels, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + +// Get the value of the tensor specified by its handle. +// +// Arguments: +// handle: The handle for a tensor stored in the session state. +// dtype: The type of the output value. +// +// Returns The tensor for the given handle. +func GetSessionTensor(scope *Scope, handle tf.Output, dtype tf.DataType) (value tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtype": dtype} + opspec := tf.OpSpec{ + Type: "GetSessionTensor", + Input: []tf.Input{ + handle, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ResourceApplyProximalGradientDescentAttr is an optional argument to ResourceApplyProximalGradientDescent. +type ResourceApplyProximalGradientDescentAttr func(optionalAttr) + +// ResourceApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value. +// +// value: If True, the subtraction will be protected by a lock; +// otherwise the behavior is undefined, but may exhibit less contention. +// If not specified, defaults to false +func ResourceApplyProximalGradientDescentUseLocking(value bool) ResourceApplyProximalGradientDescentAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// Update '*var' as FOBOS algorithm with fixed learning rate. +// +// prox_v = var - alpha * delta +// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} +// +// Arguments: +// var_: Should be from a Variable(). +// alpha: Scaling factor. Must be a scalar. +// l1: L1 regularization. Must be a scalar. +// l2: L2 regularization. Must be a scalar. +// delta: The change. +// +// Returns the created operation. +func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, delta tf.Output, optional ...ResourceApplyProximalGradientDescentAttr) (o *tf.Operation) { if scope.Err() != nil { return } @@ -18413,83 +18413,272 @@ func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output t return op.Output(0) } -// StringSplitAttr is an optional argument to StringSplit. -type StringSplitAttr func(optionalAttr) - -// StringSplitSkipEmpty sets the optional skip_empty attribute to value. -// -// value: A `bool`. If `True`, skip the empty strings from the result. -// If not specified, defaults to true -func StringSplitSkipEmpty(value bool) StringSplitAttr { - return func(m optionalAttr) { - m["skip_empty"] = value - } -} - -// Split elements of `input` based on `delimiter` into a `SparseTensor`. +// Assigns sparse updates to the variable referenced by `resource`. // -// Let N be the size of source (typically N will be the batch size). Split each -// element of `input` based on `delimiter` and return a `SparseTensor` -// containing the splitted tokens. Empty tokens are ignored. +// This operation computes // -// `delimiter` can be empty, or a string of split characters. If `delimiter` is an -// empty string, each element of `input` is split into individual single-byte -// character strings, including splitting of UTF-8 multibyte sequences. Otherwise -// every character of `delimiter` is a potential split point. +// # Scalar indices +// ref[indices, ...] = updates[...] // -// For example: -// N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output -// will be +// # Vector indices (for each i) +// ref[indices[i], ...] = updates[i, ...] // -// indices = [0, 0; -// 0, 1; -// 1, 0; -// 1, 1; -// 1, 2] -// shape = [2, 3] -// values = ['hello', 'world', 'a', 'b', 'c'] +// # High rank indices (for each i, ..., j) +// ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] // // Arguments: -// input: 1-D. Strings to split. -// delimiter: 0-D. Delimiter characters (bytes), or empty string. +// resource: Should be from a `Variable` node. +// indices: A tensor of indices into the first dimension of `ref`. +// updates: A tensor of updated values to add to `ref`. // -// Returns A dense matrix of int64 representing the indices of the sparse tensor.A vector of strings corresponding to the splited values.a length-2 vector of int64 representing the shape of the sparse -// tensor, where the first value is N and the second value is the maximum number -// of tokens in a single input entry. -func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ...StringSplitAttr) (indices tf.Output, values tf.Output, shape tf.Output) { +// Returns the created operation. +func ResourceScatterUpdate(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "StringSplit", + Type: "ResourceScatterUpdate", Input: []tf.Input{ - input, delimiter, + resource, indices, updates, }, - Attrs: attrs, } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) + return scope.AddOperation(opspec) } -// Inverse 3D real-valued fast Fourier transform. +// Given a path to new and old vocabulary files, returns a remapping Tensor of // -// Computes the inverse 3-dimensional discrete Fourier transform of a real-valued -// signal over the inner-most 3 dimensions of `input`. +// length `num_new_vocab`, where `remapping[i]` contains the row number in the old +// vocabulary that corresponds to row `i` in the new vocabulary (starting at line +// `new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i` +// in the new vocabulary is not in the old vocabulary. `num_vocab_offset` enables +// use in the partitioned variable case, and should generally be set through +// examining partitioning info. The format of the files should be a text file, +// with each line containing a single entity within the vocabulary. // -// The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`: -// The inner-most dimension contains the `fft_length / 2 + 1` unique components of -// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed -// from the size of the inner-most 3 dimensions of `input`. If the FFT length used -// to compute `input` is odd, it should be provided since it cannot be inferred -// properly. +// For example, with `new_vocab_file` a text file containing each of the following +// elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3], +// `num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be +// `[0, -1, 2]`. // -// Along each axis `IRFFT3D` is computed on, if `fft_length` (or -// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the -// corresponding dimension of `input`, the dimension is cropped. If it is larger, +// The op also returns a count of how many entries in the new vocabulary +// were present in the old vocabulary, which is used to calculate the number of +// values to initialize in a weight matrix remapping +// +// This functionality can be used to remap both row vocabularies (typically, +// features) and column vocabularies (typically, classes) from TensorFlow +// checkpoints. Note that the partitioning logic relies on contiguous vocabularies +// corresponding to div-partitioned variables. Moreover, the underlying remapping +// uses an IndexTable (as opposed to an inexact CuckooTable), so client code should +// use the corresponding index_table_from_file() as the FeatureColumn framework +// does (as opposed to tf.feature_to_id(), which uses a CuckooTable). +// +// Arguments: +// new_vocab_file: Path to the new vocab file. +// old_vocab_file: Path to the old vocab file. +// new_vocab_offset: How many entries into the new vocab file to start reading. +// num_new_vocab: Number of entries in the new vocab file to remap. +// +// Returns A Tensor of length num_new_vocab where the element at index i +// is equal to the old ID that maps to the new ID i. This element is -1 for any +// new ID that is not found in the old vocabulary.Number of new vocab entries found in old vocab. +func GenerateVocabRemapping(scope *Scope, new_vocab_file tf.Output, old_vocab_file tf.Output, new_vocab_offset int64, num_new_vocab int64) (remapping tf.Output, num_present tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"new_vocab_offset": new_vocab_offset, "num_new_vocab": num_new_vocab} + opspec := tf.OpSpec{ + Type: "GenerateVocabRemapping", + Input: []tf.Input{ + new_vocab_file, old_vocab_file, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + +// Computes softsign: `features / (abs(features) + 1)`. +func Softsign(scope *Scope, features tf.Output) (activations tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Softsign", + Input: []tf.Input{ + features, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ResizeBilinearAttr is an optional argument to ResizeBilinear. +type ResizeBilinearAttr func(optionalAttr) + +// ResizeBilinearAlignCorners sets the optional align_corners attribute to value. +// +// value: If true, rescale input by (new_height - 1) / (height - 1), which +// exactly aligns the 4 corners of images and resized images. If false, rescale +// by new_height / height. Treat similarly the width dimension. +// If not specified, defaults to false +func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr { + return func(m optionalAttr) { + m["align_corners"] = value + } +} + +// Resize `images` to `size` using bilinear interpolation. +// +// Input images can be of different types but output images are always float. +// +// Arguments: +// images: 4-D with shape `[batch, height, width, channels]`. +// size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`. The +// new size for the images. +// +// Returns 4-D with shape +// `[batch, new_height, new_width, channels]`. +func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResizeBilinear", + Input: []tf.Input{ + images, size, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ProdAttr is an optional argument to Prod. +type ProdAttr func(optionalAttr) + +// ProdKeepDims sets the optional keep_dims attribute to value. +// +// value: If true, retain reduced dimensions with length 1. +// If not specified, defaults to false +func ProdKeepDims(value bool) ProdAttr { + return func(m optionalAttr) { + m["keep_dims"] = value + } +} + +// Computes the product of elements across dimensions of a tensor. +// +// Reduces `input` along the dimensions given in `reduction_indices`. Unless +// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in +// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are +// retained with length 1. +// +// Arguments: +// input: The tensor to reduce. +// reduction_indices: The dimensions to reduce. Must be in the range +// `[-rank(input), rank(input))`. +// +// Returns The reduced tensor. +func Prod(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...ProdAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Prod", + Input: []tf.Input{ + input, reduction_indices, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// StringSplitAttr is an optional argument to StringSplit. +type StringSplitAttr func(optionalAttr) + +// StringSplitSkipEmpty sets the optional skip_empty attribute to value. +// +// value: A `bool`. If `True`, skip the empty strings from the result. +// If not specified, defaults to true +func StringSplitSkipEmpty(value bool) StringSplitAttr { + return func(m optionalAttr) { + m["skip_empty"] = value + } +} + +// Split elements of `input` based on `delimiter` into a `SparseTensor`. +// +// Let N be the size of source (typically N will be the batch size). Split each +// element of `input` based on `delimiter` and return a `SparseTensor` +// containing the splitted tokens. Empty tokens are ignored. +// +// `delimiter` can be empty, or a string of split characters. If `delimiter` is an +// empty string, each element of `input` is split into individual single-byte +// character strings, including splitting of UTF-8 multibyte sequences. Otherwise +// every character of `delimiter` is a potential split point. +// +// For example: +// N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output +// will be +// +// indices = [0, 0; +// 0, 1; +// 1, 0; +// 1, 1; +// 1, 2] +// shape = [2, 3] +// values = ['hello', 'world', 'a', 'b', 'c'] +// +// Arguments: +// input: 1-D. Strings to split. +// delimiter: 0-D. Delimiter characters (bytes), or empty string. +// +// Returns A dense matrix of int64 representing the indices of the sparse tensor.A vector of strings corresponding to the splited values.a length-2 vector of int64 representing the shape of the sparse +// tensor, where the first value is N and the second value is the maximum number +// of tokens in a single input entry. +func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ...StringSplitAttr) (indices tf.Output, values tf.Output, shape tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "StringSplit", + Input: []tf.Input{ + input, delimiter, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} + +// Inverse 3D real-valued fast Fourier transform. +// +// Computes the inverse 3-dimensional discrete Fourier transform of a real-valued +// signal over the inner-most 3 dimensions of `input`. +// +// The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`: +// The inner-most dimension contains the `fft_length / 2 + 1` unique components of +// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed +// from the size of the inner-most 3 dimensions of `input`. If the FFT length used +// to compute `input` is odd, it should be provided since it cannot be inferred +// properly. +// +// Along each axis `IRFFT3D` is computed on, if `fft_length` (or +// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the +// corresponding dimension of `input`, the dimension is cropped. If it is larger, // the dimension is padded with zeros. // // Arguments: @@ -21315,187 +21504,31 @@ func PadV2(scope *Scope, input tf.Output, paddings tf.Output, constant_values tf // // Arguments: // gradients: The backpropagated gradients to the corresponding Selu operation. -// outputs: The outputs of the corresponding Selu operation. -// -// Returns The gradients: `gradients * (outputs + scale * alpha)` -// if outputs < 0, `scale * gradients` otherwise. -func SeluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SeluGrad", - Input: []tf.Input{ - gradients, outputs, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Computes softplus: `log(exp(features) + 1)`. -func Softplus(scope *Scope, features tf.Output) (activations tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Softplus", - Input: []tf.Input{ - features, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// BatchMatMulAttr is an optional argument to BatchMatMul. -type BatchMatMulAttr func(optionalAttr) - -// BatchMatMulAdjX sets the optional adj_x attribute to value. -// -// value: If `True`, adjoint the slices of `x`. Defaults to `False`. -// If not specified, defaults to false -func BatchMatMulAdjX(value bool) BatchMatMulAttr { - return func(m optionalAttr) { - m["adj_x"] = value - } -} - -// BatchMatMulAdjY sets the optional adj_y attribute to value. -// -// value: If `True`, adjoint the slices of `y`. Defaults to `False`. -// If not specified, defaults to false -func BatchMatMulAdjY(value bool) BatchMatMulAttr { - return func(m optionalAttr) { - m["adj_y"] = value - } -} - -// Multiplies slices of two tensors in batches. -// -// Multiplies all slices of `Tensor` `x` and `y` (each slice can be -// viewed as an element of a batch), and arranges the individual results -// in a single output tensor of the same batch size. Each of the -// individual slices can optionally be adjointed (to adjoint a matrix -// means to transpose and conjugate it) before multiplication by setting -// the `adj_x` or `adj_y` flag to `True`, which are by default `False`. -// -// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]` -// and `[..., r_y, c_y]`. -// -// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where: -// -// r_o = c_x if adj_x else r_x -// c_o = r_y if adj_y else c_y -// -// It is computed as: -// -// output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :]) -// -// Arguments: -// x: 2-D or higher with shape `[..., r_x, c_x]`. -// y: 2-D or higher with shape `[..., r_y, c_y]`. -// -// Returns 3-D or higher with shape `[..., r_o, c_o]` -func BatchMatMul(scope *Scope, x tf.Output, y tf.Output, optional ...BatchMatMulAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "BatchMatMul", - Input: []tf.Input{ - x, y, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Computes softplus gradients for a softplus operation. -// -// Arguments: -// gradients: The backpropagated gradients to the corresponding softplus operation. -// features: The features passed as input to the corresponding softplus operation. -// -// Returns The gradients: `gradients / (1 + exp(-features))`. -func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SoftplusGrad", - Input: []tf.Input{ - gradients, features, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Given a path to new and old vocabulary files, returns a remapping Tensor of -// -// length `num_new_vocab`, where `remapping[i]` contains the row number in the old -// vocabulary that corresponds to row `i` in the new vocabulary (starting at line -// `new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i` -// in the new vocabulary is not in the old vocabulary. `num_vocab_offset` enables -// use in the partitioned variable case, and should generally be set through -// examining partitioning info. The format of the files should be a text file, -// with each line containing a single entity within the vocabulary. -// -// For example, with `new_vocab_file` a text file containing each of the following -// elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3], -// `num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be -// `[0, -1, 2]`. -// -// The op also returns a count of how many entries in the new vocabulary -// were present in the old vocabulary, which is used to calculate the number of -// values to initialize in a weight matrix remapping -// -// This functionality can be used to remap both row vocabularies (typically, -// features) and column vocabularies (typically, classes) from TensorFlow -// checkpoints. Note that the partitioning logic relies on contiguous vocabularies -// corresponding to div-partitioned variables. Moreover, the underlying remapping -// uses an IndexTable (as opposed to an inexact CuckooTable), so client code should -// use the corresponding index_table_from_file() as the FeatureColumn framework -// does (as opposed to tf.feature_to_id(), which uses a CuckooTable). -// -// Arguments: -// new_vocab_file: Path to the new vocab file. -// old_vocab_file: Path to the old vocab file. -// new_vocab_offset: How many entries into the new vocab file to start reading. -// num_new_vocab: Number of entries in the new vocab file to remap. -// -// Returns A Tensor of length num_new_vocab where the element at index i -// is equal to the old ID that maps to the new ID i. This element is -1 for any -// new ID that is not found in the old vocabulary.Number of new vocab entries found in old vocab. -func GenerateVocabRemapping(scope *Scope, new_vocab_file tf.Output, old_vocab_file tf.Output, new_vocab_offset int64, num_new_vocab int64) (remapping tf.Output, num_present tf.Output) { +// outputs: The outputs of the corresponding Selu operation. +// +// Returns The gradients: `gradients * (outputs + scale * alpha)` +// if outputs < 0, `scale * gradients` otherwise. +func SeluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{"new_vocab_offset": new_vocab_offset, "num_new_vocab": num_new_vocab} opspec := tf.OpSpec{ - Type: "GenerateVocabRemapping", + Type: "SeluGrad", Input: []tf.Input{ - new_vocab_file, old_vocab_file, + gradients, outputs, }, - Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) + return op.Output(0) } -// Computes softsign: `features / (abs(features) + 1)`. -func Softsign(scope *Scope, features tf.Output) (activations tf.Output) { +// Computes softplus: `log(exp(features) + 1)`. +func Softplus(scope *Scope, features tf.Output) (activations tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "Softsign", + Type: "Softplus", Input: []tf.Input{ features, }, @@ -21504,33 +21537,56 @@ func Softsign(scope *Scope, features tf.Output) (activations tf.Output) { return op.Output(0) } -// ResizeBilinearAttr is an optional argument to ResizeBilinear. -type ResizeBilinearAttr func(optionalAttr) +// BatchMatMulAttr is an optional argument to BatchMatMul. +type BatchMatMulAttr func(optionalAttr) -// ResizeBilinearAlignCorners sets the optional align_corners attribute to value. +// BatchMatMulAdjX sets the optional adj_x attribute to value. // -// value: If true, rescale input by (new_height - 1) / (height - 1), which -// exactly aligns the 4 corners of images and resized images. If false, rescale -// by new_height / height. Treat similarly the width dimension. +// value: If `True`, adjoint the slices of `x`. Defaults to `False`. // If not specified, defaults to false -func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr { +func BatchMatMulAdjX(value bool) BatchMatMulAttr { return func(m optionalAttr) { - m["align_corners"] = value + m["adj_x"] = value } } -// Resize `images` to `size` using bilinear interpolation. +// BatchMatMulAdjY sets the optional adj_y attribute to value. // -// Input images can be of different types but output images are always float. +// value: If `True`, adjoint the slices of `y`. Defaults to `False`. +// If not specified, defaults to false +func BatchMatMulAdjY(value bool) BatchMatMulAttr { + return func(m optionalAttr) { + m["adj_y"] = value + } +} + +// Multiplies slices of two tensors in batches. +// +// Multiplies all slices of `Tensor` `x` and `y` (each slice can be +// viewed as an element of a batch), and arranges the individual results +// in a single output tensor of the same batch size. Each of the +// individual slices can optionally be adjointed (to adjoint a matrix +// means to transpose and conjugate it) before multiplication by setting +// the `adj_x` or `adj_y` flag to `True`, which are by default `False`. +// +// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]` +// and `[..., r_y, c_y]`. +// +// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where: +// +// r_o = c_x if adj_x else r_x +// c_o = r_y if adj_y else c_y +// +// It is computed as: +// +// output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :]) // // Arguments: -// images: 4-D with shape `[batch, height, width, channels]`. -// size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`. The -// new size for the images. +// x: 2-D or higher with shape `[..., r_x, c_x]`. +// y: 2-D or higher with shape `[..., r_y, c_y]`. // -// Returns 4-D with shape -// `[batch, new_height, new_width, channels]`. -func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) { +// Returns 3-D or higher with shape `[..., r_o, c_o]` +func BatchMatMul(scope *Scope, x tf.Output, y tf.Output, optional ...BatchMatMulAttr) (output tf.Output) { if scope.Err() != nil { return } @@ -21539,9 +21595,9 @@ func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ... a(attrs) } opspec := tf.OpSpec{ - Type: "ResizeBilinear", + Type: "BatchMatMul", Input: []tf.Input{ - images, size, + x, y, }, Attrs: attrs, } @@ -21549,46 +21605,22 @@ func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ... return op.Output(0) } -// ProdAttr is an optional argument to Prod. -type ProdAttr func(optionalAttr) - -// ProdKeepDims sets the optional keep_dims attribute to value. -// -// value: If true, retain reduced dimensions with length 1. -// If not specified, defaults to false -func ProdKeepDims(value bool) ProdAttr { - return func(m optionalAttr) { - m["keep_dims"] = value - } -} - -// Computes the product of elements across dimensions of a tensor. -// -// Reduces `input` along the dimensions given in `reduction_indices`. Unless -// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in -// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are -// retained with length 1. +// Computes softplus gradients for a softplus operation. // // Arguments: -// input: The tensor to reduce. -// reduction_indices: The dimensions to reduce. Must be in the range -// `[-rank(input), rank(input))`. +// gradients: The backpropagated gradients to the corresponding softplus operation. +// features: The features passed as input to the corresponding softplus operation. // -// Returns The reduced tensor. -func Prod(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...ProdAttr) (output tf.Output) { +// Returns The gradients: `gradients / (1 + exp(-features))`. +func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "Prod", + Type: "SoftplusGrad", Input: []tf.Input{ - input, reduction_indices, + gradients, features, }, - Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) @@ -22842,6 +22874,76 @@ func Sqrt(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// MatrixInverseAttr is an optional argument to MatrixInverse. +type MatrixInverseAttr func(optionalAttr) + +// MatrixInverseAdjoint sets the optional adjoint attribute to value. +// If not specified, defaults to false +func MatrixInverseAdjoint(value bool) MatrixInverseAttr { + return func(m optionalAttr) { + m["adjoint"] = value + } +} + +// Computes the inverse of one or more square invertible matrices or their +// +// adjoints (conjugate transposes). +// +// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices. The output is a tensor of the same shape as the input +// containing the inverse for all input submatrices `[..., :, :]`. +// +// The op uses LU decomposition with partial pivoting to compute the inverses. +// +// If a matrix is not invertible there is no guarantee what the op does. It +// may detect the condition and raise an exception or it may simply return a +// garbage result. +// +// Arguments: +// input: Shape is `[..., M, M]`. +// +// Returns Shape is `[..., M, M]`. +// +// @compatibility(numpy) +// Equivalent to np.linalg.inv +// @end_compatibility +func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MatrixInverse", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes the gradient for the sqrt of `x` wrt its input. +// +// Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy` +// is the corresponding input gradient. +func SqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SqrtGrad", + Input: []tf.Input{ + y, dy, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Inserts a dimension of 1 into a tensor's shape. // // Given a tensor `input`, this operation inserts a dimension of 1 at the @@ -27003,73 +27105,3 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t op := scope.AddOperation(opspec) return op.Output(0) } - -// Computes the gradient for the sqrt of `x` wrt its input. -// -// Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy` -// is the corresponding input gradient. -func SqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SqrtGrad", - Input: []tf.Input{ - y, dy, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// MatrixInverseAttr is an optional argument to MatrixInverse. -type MatrixInverseAttr func(optionalAttr) - -// MatrixInverseAdjoint sets the optional adjoint attribute to value. -// If not specified, defaults to false -func MatrixInverseAdjoint(value bool) MatrixInverseAttr { - return func(m optionalAttr) { - m["adjoint"] = value - } -} - -// Computes the inverse of one or more square invertible matrices or their -// -// adjoints (conjugate transposes). -// -// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices. The output is a tensor of the same shape as the input -// containing the inverse for all input submatrices `[..., :, :]`. -// -// The op uses LU decomposition with partial pivoting to compute the inverses. -// -// If a matrix is not invertible there is no guarantee what the op does. It -// may detect the condition and raise an exception or it may simply return a -// garbage result. -// -// Arguments: -// input: Shape is `[..., M, M]`. -// -// Returns Shape is `[..., M, M]`. -// -// @compatibility(numpy) -// Equivalent to np.linalg.inv -// @end_compatibility -func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MatrixInverse", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} -- GitLab From ff5c276adf025fc498ccd81ae240bb0ba6402f3a Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 30 Oct 2017 16:17:51 -0700 Subject: [PATCH 538/573] Longer README for tf.contrib.labeled_tensor PiperOrigin-RevId: 173966577 --- tensorflow/contrib/labeled_tensor/README.md | 61 ++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/labeled_tensor/README.md b/tensorflow/contrib/labeled_tensor/README.md index 50c6750fd0..adce979e2a 100644 --- a/tensorflow/contrib/labeled_tensor/README.md +++ b/tensorflow/contrib/labeled_tensor/README.md @@ -3,6 +3,65 @@ LabeledTensor is a library for adding semantically meaningful dimension and coordinate labels to tensors in Tensorflow. -Maintainers: +LabeledTensor was inspired by [xarray](http://xarray.pydata.org) and +[pandas](http://pandas.pydata.org), projects that adds labels to NumPy array. + +## Data model + +`LabeledTensor` is an immutable object consisting of two components: + +- `tensor`: the `tf.Tensor` object containing the labeled tensor's data. +- `axes`: an OrderedDict-like object with keys given by axis names (e.g., + ``"channel"``) and values given by `Axis` objects. + +`Axis` objects keep track of the size of a dimension and, optionally, coordinate +labels along that axis (e.g., `("red", "green", "blue")`) in the form of a +tuple stored in `Axis.labels`. + +Operations on `LabeledTensors` use, preserve and transform axis names and +labels. + +## Quick start + +Try out the following snippet in a script or Jupyter notebook: + + import tensorflow as tf + + lt = tf.contrib.labeled_tensor + + # Create two LabeledTensors: + raw_image = tf.ones((299, 299, 3)) + axes = ['row', 'column', ('channel', ['red', 'green', 'blue'])] + image = lt.LabeledTensor(raw_image, axes) + assert image.tensor is raw_image + weights = lt.LabeledTensor(tf.constant([0.1, 0.3, 0.6]), + [image.axes['channel']]) + + # Examples of valid operations: + lt.transpose(image, ['column', 'row', 'channel']) + lt.reshape(image, ['row', 'column'], ['pixel']) + lt.concat([image, image], 'row') + lt.reduce_sum(image, ['channel']) + lt.select(image, {'channel': 'red'}) + lt.cast(image / 256.0, tf.uint8) + image * weights + lt.matmul(image[0, :, :], weights) + tf.cos(image) # automatically converts to tf.Tensor + +## Adding a custom op + +LabeledTensor has wrappers for [quite a +few](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/labeled_tensor/__init__.py) +TensorFlow ops. + +To easily add your own, you can use the `define_unary_op`, `define_binary_op` +and `define_reduce_op` functions, e.g., + + log = lt.define_unary_op('log', tf.log) + +## Questions + +Please reach out to the authors: + - Stephan Hoyer (shoyer@google.com, github.com/shoyer) - Eric Christiansen (ericmc@google.com, github.com/emchristiansen) -- GitLab From b46c196e9d8fa58821e3e269babe1df58d5db050 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:22:18 -0700 Subject: [PATCH 539/573] * Add graph rewrite rule that removes repeated application of scalar unary ops that are involutions (their own inverse). * Update rewrite rule for Transpose to also handle ConjugateTranspose. PiperOrigin-RevId: 173967184 --- .../optimizers/arithmetic_optimizer.cc | 20 +++++++++++++++-- .../optimizers/arithmetic_optimizer_test.cc | 22 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 400e1c017b..78b55237d1 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -31,6 +31,12 @@ namespace tensorflow { namespace grappler { namespace { +static bool IsInvolution(const NodeDef& node) { + const std::unordered_set involution_ops = {"Conj", "Reciprocal", + "Neg", "LogicalNot"}; + return involution_ops.count(node.op()) > 0; +} + bool AreInversePermutations(gtl::ArraySlice a, gtl::ArraySlice b) { if (a.size() != b.size()) { @@ -394,10 +400,20 @@ void ArithmeticOptimizer::DedupComputations(GraphDef* optimized_graph) const { string ArithmeticOptimizer::TrySimplifyAndReplaceUses( const NodeDef* node, GraphDef* graph_def, NodeMap* node_map, std::vector* new_nodes) const { + // Remove involutions applied twice. + if (IsInvolution(*node)) { + // An involution is a function f(x) that is its own inverse, + // i.e. f(f(x)) = x. + const NodeDef* input = node_map->GetNode(node->input(0)); + if (input->op() == node->op()) { + return input->input(0); + } + } + // Remove inverse transposes. - if (node->op() == "Transpose") { + if (node->op() == "Transpose" || node->op() == "ConjugateTranspose") { const NodeDef* input = node_map->GetNode(node->input(0)); - if (input->op() == "Transpose") { + if (input->op() == node->op()) { const NodeDef* node_perm = node_map->GetNode(node->input(1)); const NodeDef* input_perm = node_map->GetNode(input->input(1)); std::vector node_perm_values; diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc index 8edb34975f..61c8b82ea0 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc @@ -109,6 +109,28 @@ TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) { EXPECT_EQ("add1", new_add3.input(1)); } +TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsReal) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output c = ops::Const(s.WithOpName("c"), {1.0f, 2.0f}, {1, 2}); + Output neg1 = ops::Neg(s.WithOpName("neg1"), c); + Output neg2 = ops::Neg(s.WithOpName("neg2"), neg1); + Output recip1 = ops::Reciprocal(s.WithOpName("recip1"), neg2); + Output recip2 = ops::Reciprocal(s.WithOpName("recip2"), recip1); + Output id = ops::Identity(s.WithOpName("id"), recip2); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ArithmeticOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + EXPECT_EQ(6, output.node_size()); + EXPECT_EQ("c", output.node(1).input(0)); + EXPECT_EQ("c", output.node(3).input(0)); + EXPECT_EQ("c", output.node(5).input(0)); +} + TEST_F(ArithmeticOptimizerTest, IdentityReshape) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output inputs = -- GitLab From 0e6abfcdaf62c991ffa303454904e51ff55cf3d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:23:45 -0700 Subject: [PATCH 540/573] K-FAC: Example for multi-tower support for MNIST MLP. PiperOrigin-RevId: 173967370 --- tensorflow/contrib/kfac/examples/mlp.py | 142 +++++++++++++++--- .../contrib/kfac/examples/mlp_mnist_main.py | 11 +- .../contrib/kfac/examples/tests/mlp_test.py | 6 + .../contrib/kfac/python/kernel_tests/BUILD | 1 + 4 files changed, 137 insertions(+), 23 deletions(-) diff --git a/tensorflow/contrib/kfac/examples/mlp.py b/tensorflow/contrib/kfac/examples/mlp.py index ecebed2dd3..4275ceadc2 100644 --- a/tensorflow/contrib/kfac/examples/mlp.py +++ b/tensorflow/contrib/kfac/examples/mlp.py @@ -32,6 +32,7 @@ opt = tf.contrib.kfac.optimizer __all__ = [ "fc_layer", "train_mnist", + "train_mnist_multitower", ] @@ -60,36 +61,30 @@ def fc_layer(layer_id, inputs, output_size): activations = tf.nn.tanh(preactivations) # layer.weights is a list. This converts it a (hashable) tuple. - return preactivations, activations, tuple(layer.weights) + return preactivations, activations, (layer.kernel, layer.bias) -def train_mnist(data_dir, num_epochs, use_fake_data=False): - """Train an MLP on MNIST. +def build_model(examples, labels, num_labels, layer_collection): + """Builds an MLP classification model. Args: - data_dir: string. Directory to read MNIST examples from. - num_epochs: int. Number of passes to make over the training set. - use_fake_data: bool. If True, generate a synthetic dataset. + examples: Tensor of shape [num_examples, num_features]. Represents inputs of + model. + labels: Tensor of shape [num_examples]. Contains integer IDs to be predicted + by softmax for each example. + num_labels: int. Number of distinct values 'labels' can take on. + layer_collection: LayerCollection instance describing model architecture. Returns: - accuracy of model on the final minibatch of training data. + loss: 0-D Tensor representing loss to be minimized. + accuracy: 0-D Tensor representing model's accuracy. """ - # Load a dataset. - tf.logging.info("Loading MNIST into memory.") - examples, labels = mnist.load_mnist( - data_dir, - num_epochs=num_epochs, - batch_size=64, - flatten_images=True, - use_fake_data=use_fake_data) - # Build an MLP. For each layer, we'll keep track of the preactivations, # activations, weights, and bias. - tf.logging.info("Building model.") pre0, act0, params0 = fc_layer(layer_id=0, inputs=examples, output_size=128) pre1, act1, params1 = fc_layer(layer_id=1, inputs=act0, output_size=64) pre2, act2, params2 = fc_layer(layer_id=2, inputs=act1, output_size=32) - logits, _, params3 = fc_layer(layer_id=3, inputs=act2, output_size=10) + logits, _, params3 = fc_layer(layer_id=3, inputs=act2, output_size=num_labels) loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=logits)) @@ -99,16 +94,32 @@ def train_mnist(data_dir, num_epochs, use_fake_data=False): # Register parameters. K-FAC needs to know about the inputs, outputs, and # parameters of each layer and the logits powering the posterior probability # over classes. - tf.logging.info("Building KFAC Optimizer.") - layer_collection = lc.LayerCollection() + tf.logging.info("Building LayerCollection.") layer_collection.register_fully_connected(params0, examples, pre0) layer_collection.register_fully_connected(params1, act0, pre1) layer_collection.register_fully_connected(params2, act1, pre2) layer_collection.register_fully_connected(params3, act2, logits) - layer_collection.register_categorical_predictive_distribution(logits) + layer_collection.register_categorical_predictive_distribution( + logits, name="logits") + return loss, accuracy + + +def minimize(loss, accuracy, layer_collection, session_config=None): + """Minimize 'loss' with KfacOptimizer. + + Args: + loss: 0-D Tensor. Loss to be minimized. + accuracy: 0-D Tensor. Accuracy of classifier on current minibatch. + layer_collection: LayerCollection instance. Describes layers in model. + session_config: tf.ConfigProto. Configuration for tf.Session(). + + Returns: + accuracy of classifier on final minibatch. + """ # Train with K-FAC. We'll use a decreasing learning rate that's cut in 1/2 # every 10k iterations. + tf.logging.info("Building KFAC Optimizer.") global_step = tf.train.get_or_create_global_step() optimizer = opt.KfacOptimizer( learning_rate=tf.train.exponential_decay( @@ -120,7 +131,7 @@ def train_mnist(data_dir, num_epochs, use_fake_data=False): train_op = optimizer.minimize(loss, global_step=global_step) tf.logging.info("Starting training.") - with tf.train.MonitoredTrainingSession() as sess: + with tf.train.MonitoredTrainingSession(config=session_config) as sess: while not sess.should_stop(): # K-FAC has 3 primary ops, # - train_op: Update the weights with the minibatch's gradient. @@ -141,3 +152,90 @@ def train_mnist(data_dir, num_epochs, use_fake_data=False): global_step_, loss_, accuracy_) return accuracy_ + + +def train_mnist(data_dir, num_epochs, use_fake_data=False): + """Train an MLP on MNIST. + + Args: + data_dir: string. Directory to read MNIST examples from. + num_epochs: int. Number of passes to make over the training set. + use_fake_data: bool. If True, generate a synthetic dataset. + + Returns: + accuracy of model on the final minibatch of training data. + """ + # Load a dataset. + tf.logging.info("Loading MNIST into memory.") + examples, labels = mnist.load_mnist( + data_dir, + num_epochs=num_epochs, + batch_size=64, + flatten_images=True, + use_fake_data=use_fake_data) + + # Build an MLP. The model's layers will be added to the LayerCollection. + tf.logging.info("Building model.") + layer_collection = lc.LayerCollection() + loss, accuracy = build_model(examples, labels, 10, layer_collection) + + # Fit model. + minimize(loss, accuracy, layer_collection) + + +def train_mnist_multitower(data_dir, + num_epochs, + num_towers, + use_fake_data=False): + """Train an MLP on MNIST, splitting the minibatch across multiple towers. + + Args: + data_dir: string. Directory to read MNIST examples from. + num_epochs: int. Number of passes to make over the training set. + num_towers: int. Number of CPUs to split minibatch across. + use_fake_data: bool. If True, generate a synthetic dataset. + + Returns: + accuracy of model on the final minibatch of training data. + """ + # Load a dataset. + tower_batch_size = 64 + batch_size = tower_batch_size * num_towers + tf.logging.info( + ("Loading MNIST into memory. Using batch_size = %d = %d towers * %d " + "tower batch size.") % (batch_size, num_towers, tower_batch_size)) + examples, labels = mnist.load_mnist( + data_dir, + num_epochs=num_epochs, + batch_size=batch_size, + flatten_images=True, + use_fake_data=use_fake_data) + + # Split minibatch across towers. + examples = tf.split(examples, num_towers) + labels = tf.split(labels, num_towers) + + # Build an MLP. Each tower's layers will be added to the LayerCollection. + layer_collection = lc.LayerCollection() + tower_results = [] + for tower_id in range(num_towers): + with tf.device("/cpu:%d" % tower_id): + with tf.name_scope("tower%d" % tower_id): + with tf.variable_scope(tf.get_variable_scope(), reuse=(tower_id > 0)): + tf.logging.info("Building tower %d." % tower_id) + tower_results.append( + build_model(examples[tower_id], labels[tower_id], 10, + layer_collection)) + losses, accuracies = zip(*tower_results) + + # Average across towers. + loss = tf.reduce_mean(losses) + accuracy = tf.reduce_mean(accuracies) + + # Fit model. + session_config = tf.ConfigProto( + allow_soft_placement=False, device_count={ + "CPU": num_towers + }) + return minimize( + loss, accuracy, layer_collection, session_config=session_config) diff --git a/tensorflow/contrib/kfac/examples/mlp_mnist_main.py b/tensorflow/contrib/kfac/examples/mlp_mnist_main.py index a272f7d67a..b318c71a56 100644 --- a/tensorflow/contrib/kfac/examples/mlp_mnist_main.py +++ b/tensorflow/contrib/kfac/examples/mlp_mnist_main.py @@ -33,7 +33,11 @@ FLAGS = None def main(argv): _ = argv - mlp.train_mnist(FLAGS.data_dir, num_epochs=200) + if FLAGS.num_towers > 1: + mlp.train_mnist_multitower( + FLAGS.data_dir, num_epochs=200, num_towers=FLAGS.num_towers) + else: + mlp.train_mnist(FLAGS.data_dir, num_epochs=200) if __name__ == "__main__": @@ -43,5 +47,10 @@ if __name__ == "__main__": type=str, default="/tmp/mnist", help="Directory to store dataset in.") + parser.add_argument( + "--num_towers", + type=int, + default=1, + help="Number of CPUs to split minibatch across.") FLAGS, unparsed = parser.parse_known_args() tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/kfac/examples/tests/mlp_test.py b/tensorflow/contrib/kfac/examples/tests/mlp_test.py index 833d02baed..34a942d27f 100644 --- a/tensorflow/contrib/kfac/examples/tests/mlp_test.py +++ b/tensorflow/contrib/kfac/examples/tests/mlp_test.py @@ -47,6 +47,12 @@ class MlpTest(tf.test.TestCase): # but that takes a non-trivial amount of compute. mlp.train_mnist(data_dir=None, num_epochs=1, use_fake_data=True) + def testTrainMnistMultitower(self): + with tf.Graph().as_default(): + # Ensure model training doesn't crash. + mlp.train_mnist_multitower( + data_dir=None, num_epochs=1, num_towers=2, use_fake_data=True) + if __name__ == "__main__": tf.test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD index 0653e71d12..5d86373a23 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD +++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD @@ -88,6 +88,7 @@ py_test( deps = [ "//tensorflow/contrib/kfac/python/ops:kfac_optimizer", "//tensorflow/contrib/kfac/python/ops:layer_collection", + "//tensorflow/contrib/kfac/python/ops:loss_functions", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_ops", -- GitLab From 293ba20be14f56ccae778e3665ab999c69e2a920 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:25:14 -0700 Subject: [PATCH 541/573] Make learning_rate_decay.piecewise_constant work in Eager mode. PiperOrigin-RevId: 173967531 --- .../python/training/learning_rate_decay.py | 9 +- .../training/learning_rate_decay_test.py | 114 +++++++++--------- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py index bb7762c8c5..802b930b0e 100644 --- a/tensorflow/python/training/learning_rate_decay.py +++ b/tensorflow/python/training/learning_rate_decay.py @@ -27,6 +27,7 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops + def exponential_decay(learning_rate, global_step, decay_steps, decay_rate, staircase=False, name=None): """Applies exponential decay to the learning rate. @@ -164,13 +165,13 @@ def piecewise_constant(x, boundaries, values, name=None): raise ValueError( "Values must have elements all with the same dtype (%s vs %s)." % ( values[0].dtype.base_dtype, v.dtype.base_dtype)) - pred_fn_pairs = {} - pred_fn_pairs[x <= boundaries[0]] = lambda: values[0] - pred_fn_pairs[x > boundaries[-1]] = lambda: values[-1] + pred_fn_pairs = [] + pred_fn_pairs.append((x <= boundaries[0], lambda: values[0])) + pred_fn_pairs.append((x > boundaries[-1], lambda: values[-1])) for low, high, v in zip(boundaries[:-1], boundaries[1:], values[1:-1]): # Need to bind v here; can do this with lambda v=v: ... pred = (x > low) & (x <= high) - pred_fn_pairs[pred] = lambda v=v: v + pred_fn_pairs.append((pred, lambda v=v: v)) # The default isn't needed here because our conditions are mutually # exclusive and exhaustive, but tf.case requires it. diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py index 34c300eae7..ff41d80940 100644 --- a/tensorflow/python/training/learning_rate_decay_test.py +++ b/tensorflow/python/training/learning_rate_decay_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import math +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_state_ops @@ -43,7 +44,7 @@ class LRDecayTest(test_util.TensorFlowTestCase): def testStaircase(self): with self.test_session(): step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_100 = state_ops.assign(step, 100) assign_1 = state_ops.assign(step, 1) assign_2 = state_ops.assign(step, 2) @@ -78,65 +79,63 @@ class LRDecayTest(test_util.TensorFlowTestCase): expected = .1 * 0.96 ** (100 // 3) self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + @test_util.run_in_graph_and_eager_modes() def testPiecewiseConstant(self): - with self.test_session(): - x = variables.Variable(-999) - assign_100 = x.assign(100) - assign_105 = x.assign(105) - assign_110 = x.assign(110) - assign_120 = x.assign(120) - assign_999 = x.assign(999) - pc = learning_rate_decay.piecewise_constant(x, [100, 110, 120], - [1.0, 0.1, 0.01, 0.001]) - - variables.global_variables_initializer().run() - self.assertAllClose(pc.eval(), 1.0, 1e-6) - assign_100.op.run() - self.assertAllClose(pc.eval(), 1.0, 1e-6) - assign_105.op.run() - self.assertAllClose(pc.eval(), 0.1, 1e-6) - assign_110.op.run() - self.assertAllClose(pc.eval(), 0.1, 1e-6) - assign_120.op.run() - self.assertAllClose(pc.eval(), 0.01, 1e-6) - assign_999.op.run() - self.assertAllClose(pc.eval(), 0.001, 1e-6) - + x = resource_variable_ops.ResourceVariable(-999) + def pc(): + return learning_rate_decay.piecewise_constant(x, [100, 110, 120], + [1.0, 0.1, 0.01, 0.001]) + + self.evaluate(variables.global_variables_initializer()) + + self.assertAllClose(self.evaluate(pc()), 1.0, 1e-6) + self.evaluate(x.assign(100)) + self.assertAllClose(self.evaluate(pc()), 1.0, 1e-6) + self.evaluate(x.assign(105)) + self.assertAllClose(self.evaluate(pc()), 0.1, 1e-6) + self.evaluate(x.assign(110)) + self.assertAllClose(self.evaluate(pc()), 0.1, 1e-6) + self.evaluate(x.assign(120)) + self.assertAllClose(self.evaluate(pc()), 0.01, 1e-6) + self.evaluate(x.assign(999)) + self.assertAllClose(self.evaluate(pc()), 0.001, 1e-6) + + @test_util.run_in_graph_and_eager_modes() def testPiecewiseConstantEdgeCases(self): - with self.test_session(): - x_int = variables.Variable(0, dtype=variables.dtypes.int32) - boundaries, values = [-1.0, 1.0], [1, 2, 3] - with self.assertRaises(ValueError): - learning_rate_decay.piecewise_constant(x_int, boundaries, values) + x_int = resource_variable_ops.ResourceVariable( + 0, dtype=variables.dtypes.int32) + boundaries, values = [-1.0, 1.0], [1, 2, 3] + with self.assertRaises(ValueError): + learning_rate_decay.piecewise_constant(x_int, boundaries, values) + x = resource_variable_ops.ResourceVariable(0.0) + boundaries, values = [-1.0, 1.0], [1.0, 2, 3] + with self.assertRaises(ValueError): + learning_rate_decay.piecewise_constant(x, boundaries, values) + + # Test that ref types are valid. + if context.in_graph_mode(): x = variables.Variable(0.0) - boundaries, values = [-1.0, 1.0], [1.0, 2, 3] - with self.assertRaises(ValueError): - learning_rate_decay.piecewise_constant(x, boundaries, values) - - # Test that ref types are valid. x_ref = x.op.outputs[0] # float32_ref tensor should be accepted boundaries, values = [1.0, 2.0], [1, 2, 3] learning_rate_decay.piecewise_constant(x_ref, boundaries, values) - # Test casting boundaries from int32 to int64. - x_int64 = variables.Variable(0, dtype=variables.dtypes.int64) - assign_1 = x_int64.assign(1) - assign_2 = x_int64.assign(2) - assign_3 = x_int64.assign(3) - assign_4 = x_int64.assign(4) - boundaries, values = [1, 2, 3], [0.4, 0.5, 0.6, 0.7] - pc = learning_rate_decay.piecewise_constant(x_int64, boundaries, values) - - variables.global_variables_initializer().run() - self.assertAllClose(pc.eval(), 0.4, 1e-6) - assign_1.op.run() - self.assertAllClose(pc.eval(), 0.4, 1e-6) - assign_2.op.run() - self.assertAllClose(pc.eval(), 0.5, 1e-6) - assign_3.op.run() - self.assertAllClose(pc.eval(), 0.6, 1e-6) - assign_4.op.run() - self.assertAllClose(pc.eval(), 0.7, 1e-6) + # Test casting boundaries from int32 to int64. + x_int64 = resource_variable_ops.ResourceVariable( + 0, dtype=variables.dtypes.int64) + boundaries, values = [1, 2, 3], [0.4, 0.5, 0.6, 0.7] + def pc(): + return learning_rate_decay.piecewise_constant(x_int64, boundaries, values) + + self.evaluate(variables.global_variables_initializer()) + self.assertAllClose(self.evaluate(pc()), 0.4, 1e-6) + self.evaluate(x_int64.assign(1)) + self.assertAllClose(self.evaluate(pc()), 0.4, 1e-6) + self.evaluate(x_int64.assign(2)) + self.assertAllClose(self.evaluate(pc()), 0.5, 1e-6) + self.evaluate(x_int64.assign(3)) + self.assertAllClose(self.evaluate(pc()), 0.6, 1e-6) + self.evaluate(x_int64.assign(4)) + self.assertAllClose(self.evaluate(pc()), 0.7, 1e-6) class LinearDecayTest(test_util.TensorFlowTestCase): @@ -245,6 +244,7 @@ class SqrtDecayTest(test_util.TensorFlowTestCase): expected = (lr - end_lr) * 0.25 ** power + end_lr self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + class PolynomialDecayTest(test_util.TensorFlowTestCase): def testBeginWithCycle(self): @@ -265,7 +265,7 @@ class ExponentialDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr, step, @@ -282,7 +282,7 @@ class ExponentialDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr, @@ -305,7 +305,7 @@ class InverseDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.inverse_time_decay(initial_lr, @@ -324,7 +324,7 @@ class InverseDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.inverse_time_decay(initial_lr, -- GitLab From c315cf1ee61d5d302c7970f92c1cc76e94f0a242 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Mon, 30 Oct 2017 16:33:17 -0700 Subject: [PATCH 542/573] Internal-only changes PiperOrigin-RevId: 173968246 --- tensorflow/contrib/eager/python/tfe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 4164a815cd..b6c687c829 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -48,7 +48,6 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@Iterator @@Network @@Saver -@@SummaryWriter @@restore_variables_on_create @@Variable @@get_optimizer_variables @@ -78,7 +77,6 @@ from tensorflow.contrib.eager.python.network import Network from tensorflow.contrib.eager.python.saver import get_optimizer_variables from tensorflow.contrib.eager.python.saver import restore_variables_on_create from tensorflow.contrib.eager.python.saver import Saver -from tensorflow.contrib.eager.python.summary_writer import SummaryWriter from tensorflow.python.eager import backprop from tensorflow.python.eager import function from tensorflow.python.eager.context import DEVICE_PLACEMENT_EXPLICIT -- GitLab From 09f62ab38b82be7ea5bc01e253d61a185a877fb8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:54:23 -0700 Subject: [PATCH 543/573] Speeding up the case for sparse float columns that have only 1 value. PiperOrigin-RevId: 173971121 --- .../contrib/boosted_trees/lib/utils/example.h | 114 +++++++++++------- .../boosted_trees/lib/utils/example_test.cc | 53 +++++--- .../lib/utils/examples_iterable.cc | 4 + .../lib/utils/examples_iterable.h | 55 ++++++--- .../lib/utils/examples_iterable_test.cc | 1 + 5 files changed, 148 insertions(+), 79 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example.h b/tensorflow/contrib/boosted_trees/lib/utils/example.h index 9514416660..e388cf332c 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/example.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/example.h @@ -17,7 +17,6 @@ #define THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ #include -#include #include #include #include "tensorflow/contrib/boosted_trees/lib/utils/optional_value.h" @@ -25,55 +24,85 @@ namespace tensorflow { namespace boosted_trees { namespace utils { - -// A matrix that given feature column id and feature value id will return -// either a value or an optional. First index indicates feature column, second -// index - the index of the value within this column - for single valued, it -// will be 0. -// Allows double-subscript access [][]. +// Represents sparse vector that have a value for some feature indices within +// the feature column. +// Allows subscript access []. template -class SparseMatrix { - typedef std::vector> SparseMap; - - class Proxy { - public: - Proxy(const int32 feature_column_idx, const SparseMap& values) - : feature_column_idx_(feature_column_idx), values_(values) {} - - OptionalValue operator[](int feature_idx) const { - auto value_iter = std::find_if( - values_.begin(), values_.end(), - [this, &feature_idx](const std::tuple& element) { - return std::get<0>(element) == feature_column_idx_ && - std::get<1>(element) == feature_idx; - }); - - if (value_iter == values_.end()) { - return OptionalValue(); - } - // There is this feature column and feature id. - return OptionalValue(std::get<2>(*value_iter)); +class SparseMultidimensionalValues { + public: + void Add(const int32 feature_idx, const T value) { + values_.emplace_back(feature_idx, value); + } + + void Clear() { values_.clear(); } + + void Reserve(const int32 size) { values_.reserve(size); } + + OptionalValue operator[](int feature_idx) const { + auto value_iter = + std::find_if(values_.begin(), values_.end(), + [&feature_idx](const std::pair& element) { + return element.first == feature_idx; + }); + + if (value_iter == values_.end()) { + return OptionalValue(); } + return OptionalValue(value_iter->second); + } - private: - int32 feature_column_idx_; - const SparseMap& values_; - }; + private: + std::vector> values_; +}; +// Represents storage for a sparse float feature column. Can store values either +// for one dimensional or a multivalent (multidimensional) sparse column. +// Allows subscript operator access [feature_id]. +template +class SparseFloatFeatureColumn { public: - void addElement(const int32 feature_column_idx, const int32 feature_idx, - const T value) { - values_.emplace_back(feature_column_idx, feature_idx, value); + void Reserve(const int32 size) { + if (!single_dimensional_) { + mutlidimensional_values.Reserve(size); + } + } + + void SetDimension(const int32 dimension) { + single_dimensional_ = dimension <= 1; + } + + void Add(const int32 feature_idx, const float value) { + if (single_dimensional_) { + DCHECK_EQ(0, feature_idx); + single_value_ = value; + } else { + mutlidimensional_values.Add(feature_idx, value); + } + initialized_ = true; } - void clear() { values_.clear(); } + void Clear() { + single_dimensional_ = false; + initialized_ = false; + mutlidimensional_values.Clear(); + } - Proxy operator[](int feature_column_idx) const { - return Proxy(feature_column_idx, values_); + OptionalValue operator[](int feature_idx) const { + if (!initialized_) { + return OptionalValue(); + } + if (single_dimensional_) { + return OptionalValue(single_value_); + } else { + return mutlidimensional_values[feature_idx]; + } } private: - SparseMap values_; + bool single_dimensional_; + bool initialized_; + T single_value_; + SparseMultidimensionalValues mutlidimensional_values; }; // Holds data for one example and enables lookup by feature column. @@ -87,9 +116,10 @@ struct Example { // Dense and sparse float features indexed by feature column. // TODO(salehay): figure out a design to support multivalent float features. std::vector dense_float_features; - // Sparse float features are allowed to be multivalent and thus can be - // represented as a sparse matrix. - SparseMatrix sparse_float_features; + + // Sparse float features columns (can be either single or multivalent + // (multidimensional). + std::vector> sparse_float_features; // Sparse integer features indexed by feature column. // Note that all integer features are assumed to be categorical, i.e. will diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc index f78fd25022..be9d63ee8a 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc @@ -25,21 +25,33 @@ namespace { class ExampleTest : public ::testing::Test {}; TEST_F(ExampleTest, TestSparseMatrix) { - // Create the following matrix: - // row id | | 0.4 | 0.3 - // 0 | 1 | | 2 - // 1 | 3 | 1 | 5 - // 2 | | | -4 - // 3 | | | - SparseMatrix matrix; - matrix.addElement(0, 1, 0.4f); - matrix.addElement(0, 2, 0.3f); - matrix.addElement(1, 0, 1.f); - matrix.addElement(1, 2, 2.f); - matrix.addElement(2, 0, 3.f); - matrix.addElement(2, 1, 1.f); - matrix.addElement(2, 2, 5.f); - matrix.addElement(3, 2, -4.f); + // Create the following matrix (FC is feature column): + // FC | f0 | f1 | f2 + // multidimensional + // 0 | | 0.4 | 0.3 + // 1 | 1 | | 2 + // 2 | 3 | 1 | 5 + // 3 | | | + // one dimensional columns + // 4 | -4 + // 5 | + std::vector> matrix; + matrix.resize(6); + matrix[0].SetDimension(3); + matrix[1].SetDimension(3); + matrix[2].SetDimension(3); + matrix[3].SetDimension(3); + matrix[4].SetDimension(1); + matrix[5].SetDimension(1); + + matrix[0].Add(1, 0.4f); + matrix[0].Add(2, 0.3f); + matrix[1].Add(0, 1.f); + matrix[1].Add(2, 2.f); + matrix[2].Add(0, 3.f); + matrix[2].Add(1, 1.f); + matrix[2].Add(2, 5.f); + matrix[4].Add(0, -4.f); // Row 0. EXPECT_FALSE(matrix[0][0].has_value()); @@ -66,13 +78,14 @@ TEST_F(ExampleTest, TestSparseMatrix) { // Row 3. EXPECT_FALSE(matrix[3][0].has_value()); EXPECT_FALSE(matrix[3][1].has_value()); - EXPECT_TRUE(matrix[3][2].has_value()); - EXPECT_EQ(-4.f, matrix[3][2].get_value()); + EXPECT_FALSE(matrix[3][2].has_value()); // Row 4. - EXPECT_FALSE(matrix[4][0].has_value()); - EXPECT_FALSE(matrix[4][1].has_value()); - EXPECT_FALSE(matrix[4][2].has_value()); + EXPECT_TRUE(matrix[4][0].has_value()); + EXPECT_EQ(-4.f, matrix[4][0].get_value()); + + // Row 5. + EXPECT_FALSE(matrix[5][0].has_value()); } } // namespace diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc index 3b287b1dcf..e7e0b568c6 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc @@ -36,12 +36,14 @@ ExamplesIterable::ExamplesIterable( // Create sparse float column iterables and values. sparse_float_column_iterables_.reserve(sparse_float_feature_columns.size()); sparse_float_column_values_.reserve(sparse_float_feature_columns.size()); + sparse_float_dimensions_.reserve(sparse_float_feature_columns.size()); for (auto& sparse_float_column : sparse_float_feature_columns) { sparse_float_column_iterables_.emplace_back( sparse_float_column.indices().template matrix(), example_start, example_end); sparse_float_column_values_.emplace_back( sparse_float_column.values().template vec()); + sparse_float_dimensions_.push_back(sparse_float_column.shape()[1]); } // Create sparse int column iterables and values. @@ -74,6 +76,8 @@ Iterator::Iterator(ExamplesIterable* iter, int64 example_idx) example_.dense_float_features.resize( iter_->dense_float_column_values_.size()); example_.sparse_int_features.resize(iter_->sparse_int_column_values_.size()); + example_.sparse_float_features.resize( + iter_->sparse_float_column_values_.size()); } } // namespace utils diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h index 72b7486872..5b33c81588 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h @@ -87,33 +87,51 @@ class ExamplesIterable { // Get sparse float values per column. auto& sparse_float_features = example_.sparse_float_features; - sparse_float_features.clear(); // Iterate through each sparse float feature column. for (size_t sparse_float_idx = 0; sparse_float_idx < iter_->sparse_float_column_iterables_.size(); ++sparse_float_idx) { + // Clear info from a previous instance. + sparse_float_features[sparse_float_idx].Clear(); + // Get range for values tensor. const auto& row_range = (*sparse_float_column_iterators_[sparse_float_idx]); DCHECK_EQ(example_idx_, row_range.example_idx); + // If the example has this feature column. if (row_range.start < row_range.end) { - // Retrieve original indices tensor. - const TTypes::ConstMatrix& indices = - iter_->sparse_float_column_iterables_[sparse_float_idx] - .sparse_indices(); - - // For each value. - for (int64 row_idx = row_range.start; row_idx < row_range.end; - ++row_idx) { - // Get the feature id for the feature column and the value. - const int32 feature_id = indices(row_idx, 1); - DCHECK_EQ(example_idx_, indices(row_idx, 0)); - - // Save the value to our sparse matrix. - sparse_float_features.addElement( - sparse_float_idx, feature_id, - iter_->sparse_float_column_values_[sparse_float_idx](row_idx)); + const int32 dimension = + iter_->sparse_float_dimensions_[sparse_float_idx]; + sparse_float_features[sparse_float_idx].SetDimension(dimension); + if (dimension <= 1) { + // single dimensional sparse feature column. + DCHECK_EQ(1, row_range.end - row_range.start); + sparse_float_features[sparse_float_idx].Add( + 0, iter_->sparse_float_column_values_[sparse_float_idx]( + row_range.start)); + } else { + // Retrieve original indices tensor. + const TTypes::ConstMatrix& indices = + iter_->sparse_float_column_iterables_[sparse_float_idx] + .sparse_indices(); + + sparse_float_features[sparse_float_idx].Reserve(row_range.end - + row_range.start); + + // For each value. + for (int64 row_idx = row_range.start; row_idx < row_range.end; + ++row_idx) { + // Get the feature id for the feature column and the value. + const int32 feature_id = indices(row_idx, 1); + DCHECK_EQ(example_idx_, indices(row_idx, 0)); + + // Save the value to our sparse matrix. + sparse_float_features[sparse_float_idx].Add( + feature_id, + iter_->sparse_float_column_values_[sparse_float_idx]( + row_idx)); + } } } } @@ -173,6 +191,9 @@ class ExamplesIterable { // Sparse float column values. std::vector::ConstVec> sparse_float_column_values_; + // Dimensions for sparse float feature columns. + std::vector sparse_float_dimensions_; + // Sparse int column iterables. std::vector sparse_int_column_iterables_; diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc index 05c166edc6..d8a6088648 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc @@ -194,6 +194,7 @@ TEST_F(ExamplesIterableTest, Iterate) { {dense_float_tensor}, {sparse_float_tensor1, sparse_float_tensor2}, {sparse_int_tensor1, sparse_int_tensor2}, 0, 8); int64 example_idx = 0; + for (const auto& example : full_iterable) { validate_example_features(example_idx, example); ++example_idx; -- GitLab From 72be26dc821c536cc7c16740166cebb1c9fb3efa Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:56:03 -0700 Subject: [PATCH 544/573] [tf.data] Iterator Save and Restore for Dataset.from_tensors(..), Dataset.from_tensor_slices(..) and dataset.concatenate(..). PiperOrigin-RevId: 173971324 --- .../contrib/data/python/kernel_tests/BUILD | 12 ++ .../concatenate_dataset_op_test.py | 138 ++++++++++++++++++ .../dataset_constructor_op_test.py | 133 +++++++++++++++++ .../core/kernels/concatenate_dataset_op.cc | 47 +++++- tensorflow/core/kernels/dataset.h | 63 ++++++-- tensorflow/core/kernels/iterator_ops.cc | 2 +- tensorflow/core/kernels/tensor_dataset_op.cc | 38 ++++- .../core/kernels/tensor_slice_dataset_op.cc | 41 +++++- 8 files changed, 450 insertions(+), 24 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index ff59e80b79..22a027f178 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -74,9 +74,12 @@ py_test( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", "//tensorflow/python:tensor_shape", + "//tensorflow/python:training", "//tensorflow/python/data/util:nest", "//third_party/py/numpy", ], @@ -93,6 +96,7 @@ py_test( ], deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -104,6 +108,7 @@ py_test( "//tensorflow/python:resource_variable_ops", "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", + "//tensorflow/python:training", "//tensorflow/python/data/util:nest", "//third_party/py/numpy", ], @@ -241,6 +246,7 @@ py_test( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -248,6 +254,7 @@ py_test( "//tensorflow/python:data_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", "//tensorflow/python:functional_ops", "//tensorflow/python:io_ops", "//tensorflow/python:lookup_ops", @@ -255,6 +262,7 @@ py_test( "//tensorflow/python:random_ops", "//tensorflow/python:script_ops", "//tensorflow/python:string_ops", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//third_party/py/numpy", @@ -396,10 +404,14 @@ py_test( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:training", + "//tensorflow/python/data/util:nest", "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py index a77f3232ce..870352209a 100644 --- a/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py @@ -17,13 +17,17 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import numpy as np from tensorflow.contrib.data.python.ops import dataset_ops +from tensorflow.contrib.data.python.ops import iterator_ops from tensorflow.python.data.util import nest from tensorflow.python.framework import errors +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib class ConcatenateDatasetTest(test.TestCase): @@ -129,6 +133,140 @@ class ConcatenateDatasetTest(test.TestCase): with self.assertRaisesRegexp(TypeError, "have different types"): input_dataset.concatenate(dataset_to_concatenate) + def _iterator_checkpoint_prefix(self): + return os.path.join(self.get_temp_dir(), "iterator") + + def _build_graph(self, input_components, to_concatenate_components): + input_dataset = dataset_ops.Dataset.from_tensor_slices(input_components) + dataset_to_concatenate = dataset_ops.Dataset.from_tensor_slices( + to_concatenate_components) + iterator = input_dataset.concatenate( + dataset_to_concatenate).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + saveable = iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + # TODO(shivaniagrawal) : non-intuitive way, add support in mata_graph + for t in nest.flatten(get_next): + ops.add_to_collection("get_next", t) + return init_op, get_next + + def _testSaveRestoreUtility(self, start, break_range, stop): + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + input_components = (np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 4)) + to_concatenate_components = (np.tile( + np.array([[5], [6], [7], [8], [9]]), 20), np.tile( + np.array([[16], [17], [18], [19], [20]]), 15)) + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph(input_components, + to_concatenate_components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(start, break_range): + result = sess.run(get_next) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_range, stop): + result = sess.run(get_next) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testRestoreAtFirstDataset(self): + start = 0 + stop = 9 + break_range = 3 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreAtSecondDataset(self): + start = 0 + stop = 9 + break_range = 6 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreAtBetweenDatasets(self): + start = 0 + stop = 9 + break_range = 4 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreExhaustedIterator(self): + start = 0 + stop = 9 + break_range = 9 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreInModifiedGraph(self): + start = 0 + stop = 9 + break_range = 6 + path = self._iterator_checkpoint_prefix() + step = 0 + + input_components = (np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 4)) + to_concatenate_components = (np.tile( + np.array([[5], [6], [7], [8], [9]]), 20), np.tile( + np.array([[16], [17], [18], [19], [20]]), 15)) + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph(input_components, + to_concatenate_components) + saver = saver_lib.Saver(allow_empty=True) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(start, break_range): + result = sess.run(get_next) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + saver.save(sess, path, step) + + new_to_concatenate_components = (np.array([[5], [6], [7], [8], [9]]), + np.array([[16], [17], [18], [19], [20]])) + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph(input_components, + new_to_concatenate_components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_range, stop): + result = sess.run(get_next) + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py b/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py index a66714feda..c3d6bfc097 100644 --- a/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py @@ -17,12 +17,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import threading import numpy as np from tensorflow.contrib.data.python.ops import batching from tensorflow.contrib.data.python.ops import dataset_ops +from tensorflow.contrib.data.python.ops import iterator_ops from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session from tensorflow.python.data.util import nest @@ -34,6 +36,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib class DatasetConstructorTest(test.TestCase): @@ -571,6 +574,136 @@ class DatasetConstructorTest(test.TestCase): new = batching._RestructuredDataset(dataset, new_types, new_shape_lists) # pylint: enable=protected-access + def _iterator_checkpoint_prefix(self): + return os.path.join(self.get_temp_dir(), "iterator") + + def _testSaveRestoreFromTensorsUtility(self, start, break_range, stop): + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) + + with ops.Graph().as_default() as g: + iterator = ( + dataset_ops.Dataset.from_tensors(components) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + saveable = iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + for t in nest.flatten(get_next): + ops.add_to_collection("get_next", t) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(start, break_range): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component, result_component) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b", "c"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for _ in range(break_range, stop): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component, result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testRestoreFromTensors(self): + self._testSaveRestoreFromTensorsUtility(0, 0, 1) + + def testRestoreExhuatedIteratorFromTensors(self): + self._testSaveRestoreFromTensorsUtility(0, 1, 1) + + def _build_graph_tensor_slices(self, components): + iterator = dataset_ops.Dataset.from_tensor_slices( + components).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + saveable = iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + for t in nest.flatten(get_next): + ops.add_to_collection("get_next", t) + return init_op, get_next + + def _testSaveRestoreFromTensorSlicesUtility(self, start, break_range, stop): + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + components = (np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 22), + np.array([37.0, 38.0, 39.0, 40.0])) + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph_tensor_slices(components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(start, break_range): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component[i], result_component) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b", "c"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_range, stop): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component[i], result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testRestoreFromTensorSlices(self): + self._testSaveRestoreFromTensorSlicesUtility(0, 4, 2) + + def testRestoreExhaustedIteratorFromTensorSlices(self): + self._testSaveRestoreFromTensorSlicesUtility(0, 4, 4) + + def tesRestoreFromTensorSlicesWithDict(self): + + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]} + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph_tensor_slices(components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(2): + results = sess.run(get_next) + self.assertEqual(components["foo"][i], results["foo"]) + self.assertEqual(components["bar"][i], results["bar"]) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(2, 3): + results = sess.run(get_next) + self.assertEqual(components["foo"][i], results["foo"]) + self.assertEqual(components["bar"][i], results["bar"]) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + if __name__ == "__main__": test.main() diff --git a/tensorflow/core/kernels/concatenate_dataset_op.cc b/tensorflow/core/kernels/concatenate_dataset_op.cc index a6d27852b5..711c234129 100644 --- a/tensorflow/core/kernels/concatenate_dataset_op.cc +++ b/tensorflow/core/kernels/concatenate_dataset_op.cc @@ -36,15 +36,17 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel { " have different output_types %s and %s", (DataTypeVectorString(input->output_dtypes()), DataTypeVectorString(to_concatenate->output_dtypes())))); - *output = new Dataset(input, to_concatenate); + *output = new Dataset(ctx, input, to_concatenate); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - explicit Dataset(const DatasetBase* input, + explicit Dataset(OpKernelContext* ctx, const DatasetBase* input, const DatasetBase* to_concatenate) - : input_(input), to_concatenate_(to_concatenate) { + : GraphDatasetBase(ctx), + input_(input), + to_concatenate_(to_concatenate) { input_->Ref(); to_concatenate_->Ref(); @@ -76,6 +78,19 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel { string DebugString() override { return "ConcatenateDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + Node* input_graph = nullptr; + TF_RETURN_IF_ERROR(b->AddParentDataset(input_, &input_graph)); + Node* to_concatenate_graph = nullptr; + TF_RETURN_IF_ERROR( + b->AddParentDataset(to_concatenate_, &to_concatenate_graph)); + TF_RETURN_IF_ERROR( + b->AddDataset(this, {input_graph, to_concatenate_graph}, output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -105,6 +120,30 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel { return Status::OK(); } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); + TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_)); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); + if (!TF_PREDICT_TRUE(i_ >= 0 && i_ <= 2)) + return errors::InvalidArgument("i_ must be in range [0, 2]."); + if (i_ == 1) { + input_impl_ = dataset()->to_concatenate_->MakeIterator( + strings::StrCat(prefix(), "[1]")); + } else if (i_ == 2) { + input_impl_.reset(); + } + TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_)); + return Status::OK(); + } + private: mutex mu_; int64 i_ GUARDED_BY(mu_); diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h index a431889409..e0ffe268dd 100644 --- a/tensorflow/core/kernels/dataset.h +++ b/tensorflow/core/kernels/dataset.h @@ -56,7 +56,7 @@ class IteratorStateReader { // Used for saving iterator state. class IteratorStateWriter { public: - virtual Status WriteScalar(StringPiece key, const int64& val) = 0; + virtual Status WriteScalar(StringPiece key, const int64 val) = 0; virtual Status WriteScalar(StringPiece key, const string& val) = 0; virtual ~IteratorStateWriter() {} @@ -75,10 +75,7 @@ class GraphDefBuilderWrapper { Status AddScalar(const T& val, Node** output) { Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); val_t.scalar()() = val; - *output = - ops::SourceOp("Const", b_->opts() - .WithAttr("dtype", DataTypeToEnum::v()) - .WithAttr("value", val_t)); + AddTensorInternal(val_t, output); if (*output == nullptr) { return errors::Internal("AddScalar: Failed to build Const op."); } @@ -96,16 +93,25 @@ class GraphDefBuilderWrapper { for (int i = 0; i < val.size(); i++) { val_t.flat()(i) = val[i]; } - *output = - ops::SourceOp("Const", b_->opts() - .WithAttr("dtype", DataTypeToEnum::v()) - .WithAttr("value", val_t)); + AddTensorInternal(val_t, output); if (*output == nullptr) { return errors::Internal("AddVector: Failed to build Const op."); } return Status::OK(); } + // Adds a Const node with Tensor value to the Graph. + // `*output` contains a pointer to the output `Node`. It is guaranteed to be + // non-null if the method returns with an OK status. + // The returned Node pointer is owned by the backing Graph of GraphDefBuilder. + Status AddTensor(const Tensor& val, Node** output) { + AddTensorInternal(val, output); + if (*output == nullptr) { + return errors::Internal("AddTesor: Failed to build Const op."); + } + return Status::OK(); + } + // Adds a node corresponding to the `DatasetType` to the Graph. // Return value of `DatasetType::op_name()` is used as the op type for the // node. @@ -148,7 +154,46 @@ class GraphDefBuilderWrapper { return Status::OK(); } + // TODO(shivaniagrawal): Single method for AddDataset for + // NodeOut/ArrraySlice + template + Status AddDatasetWithInputAsList(const DatasetType* dataset, + gtl::ArraySlice input, + Node** output) { + const string& op_type_name = dataset->op_name(); + std::unique_ptr opts( + new GraphDefBuilder::Options(b_->opts())); + bool has_output_types_attr = HasAttr(op_type_name, "output_types"); + bool has_output_shapes_attr = HasAttr(op_type_name, "output_shapes"); + if (has_output_shapes_attr) { + opts.reset(new GraphDefBuilder::Options( + opts->WithAttr("output_shapes", dataset->output_shapes()))); + } + if (has_output_types_attr) { + opts.reset(new GraphDefBuilder::Options( + opts->WithAttr("output_types", dataset->output_dtypes()))); + } + if (opts->HaveError()) { + return errors::Internal("AddDataset: Error building Options."); + } + NodeBuilder node_builder(opts->GetNameForOp(op_type_name), op_type_name, + opts->op_registry()); + node_builder.Input(input); + *output = opts->FinalizeBuilder(&node_builder); + if (*output == nullptr) { + return errors::Internal("AddDataset: Failed to build ", op_type_name, + " op."); + } + return Status::OK(); + } + private: + void AddTensorInternal(const Tensor& val, Node** output) { + *output = ops::SourceOp( + "Const", + b_->opts().WithAttr("dtype", val.dtype()).WithAttr("value", val)); + } + bool HasAttr(const string& op_type_name, const string& attr_name) { const OpDef* op_def = nullptr; Status s = b_->opts().op_registry()->LookUpOpDef(op_type_name, &op_def); diff --git a/tensorflow/core/kernels/iterator_ops.cc b/tensorflow/core/kernels/iterator_ops.cc index b7c1fff2a9..d8bcd09842 100644 --- a/tensorflow/core/kernels/iterator_ops.cc +++ b/tensorflow/core/kernels/iterator_ops.cc @@ -228,7 +228,7 @@ class VariantTensorDataWriter : public IteratorStateWriter { // Does not take ownership of data. explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {} - Status WriteScalar(StringPiece key, const int64& val) override { + Status WriteScalar(StringPiece key, const int64 val) override { return WriteScalarInternal(key, val); } diff --git a/tensorflow/core/kernels/tensor_dataset_op.cc b/tensorflow/core/kernels/tensor_dataset_op.cc index 36caf965d7..db7c947328 100644 --- a/tensorflow/core/kernels/tensor_dataset_op.cc +++ b/tensorflow/core/kernels/tensor_dataset_op.cc @@ -40,14 +40,14 @@ class TensorDatasetOp : public DatasetOpKernel { for (const Tensor& t : inputs) { components.push_back(t); } - *output = new Dataset(std::move(components)); + *output = new Dataset(ctx, std::move(components)); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - explicit Dataset(std::vector tensors) - : tensors_(std::move(tensors)) { + Dataset(OpKernelContext* ctx, std::vector tensors) + : GraphDatasetBase(ctx), tensors_(std::move(tensors)) { for (const Tensor& t : tensors_) { dtypes_.push_back(t.dtype()); shapes_.emplace_back(t.shape().dim_sizes()); @@ -67,6 +67,21 @@ class TensorDatasetOp : public DatasetOpKernel { string DebugString() override { return "TensorDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + std::vector components; + components.reserve(tensors_.size()); + for (const Tensor& t : tensors_) { + Node* node; + TF_RETURN_IF_ERROR(b->AddTensor(t, &node)); + components.emplace_back(node); + } + TF_RETURN_IF_ERROR( + b->AddDatasetWithInputAsList(this, components, output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -88,6 +103,21 @@ class TensorDatasetOp : public DatasetOpKernel { } } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + if (produced_) + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("produced"), "")); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + produced_ = reader->Contains(full_name("produced")); + return Status::OK(); + } + private: mutex mu_; bool produced_ GUARDED_BY(mu_); diff --git a/tensorflow/core/kernels/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/tensor_slice_dataset_op.cc index 7b652401bc..fd36bf524c 100644 --- a/tensorflow/core/kernels/tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/tensor_slice_dataset_op.cc @@ -50,14 +50,14 @@ class TensorSliceDatasetOp : public DatasetOpKernel { errors::InvalidArgument( "All components must have the same size in the 0th dimension")); } - *output = new Dataset(std::move(components)); + *output = new Dataset(ctx, std::move(components)); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - explicit Dataset(std::vector tensors) - : tensors_(std::move(tensors)) { + explicit Dataset(OpKernelContext* ctx, std::vector tensors) + : GraphDatasetBase(ctx), tensors_(std::move(tensors)) { for (const Tensor& t : tensors_) { dtypes_.push_back(t.dtype()); gtl::InlinedVector partial_dim_sizes; @@ -83,6 +83,21 @@ class TensorSliceDatasetOp : public DatasetOpKernel { string DebugString() override { return "TensorSliceDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + std::vector components; + components.reserve(tensors_.size()); + for (const Tensor& t : tensors_) { + Node* node; + TF_RETURN_IF_ERROR(b->AddTensor(t, &node)); + components.emplace_back(node); + } + TF_RETURN_IF_ERROR( + b->AddDatasetWithInputAsList(this, components, output)); + return Status::OK(); + } + private: template static Status HandleSliceToElement(const Tensor& parent, Tensor* element, @@ -148,10 +163,24 @@ class TensorSliceDatasetOp : public DatasetOpKernel { return Status::OK(); } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); + return Status::OK(); + } + private: mutex mu_; - int i_ GUARDED_BY(mu_); - const int n_; + int64 i_ GUARDED_BY(mu_); + const int64 n_; }; const std::vector tensors_; -- GitLab From 73fdaf0b560dd086d60a1e053affc5bfeed00097 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 16:58:34 -0700 Subject: [PATCH 545/573] Summary-writing support for Evaluators. PiperOrigin-RevId: 173971621 --- tensorflow/contrib/eager/python/BUILD | 6 +++ tensorflow/contrib/eager/python/evaluator.py | 40 +++++++++++++++---- .../contrib/eager/python/evaluator_test.py | 37 ++++++++++++++++- .../contrib/eager/python/metrics_test.py | 17 ++------ tensorflow/contrib/summary/summary.py | 1 + tensorflow/contrib/summary/summary_ops.py | 7 ++++ 6 files changed, 87 insertions(+), 21 deletions(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 614a080e61..2b84bc2e9b 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -164,6 +164,7 @@ py_test( deps = [ ":metrics", "//tensorflow/contrib/summary:summary_ops", + "//tensorflow/contrib/summary:summary_test_util", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:dtypes", @@ -185,6 +186,7 @@ py_library( deps = [ ":datasets", ":metrics", + "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", @@ -201,6 +203,10 @@ py_test( deps = [ ":evaluator", ":metrics", + "//tensorflow/contrib/summary:summary_test_util", + "//tensorflow/python:framework_ops", + "//tensorflow/python:training", + "//tensorflow/python:variables", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py index 633c747e5e..bd0ab02ecf 100644 --- a/tensorflow/contrib/eager/python/evaluator.py +++ b/tensorflow/contrib/eager/python/evaluator.py @@ -22,6 +22,7 @@ import six from tensorflow.contrib.eager.python import datasets from tensorflow.contrib.eager.python import metrics +from tensorflow.contrib.summary import summary_ops from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import errors_impl @@ -36,7 +37,7 @@ class Evaluator(object): evaluator = my_model.evaluator() # or MyEvaluator(my_model) for example_batch in ...: evaluator(example_batch) - results = evaluator.all_metric_results(optional_summary_writer) + results = evaluator.all_metric_results(optional_summary_logdir) Or, if you are getting your examples from a tf.data.Dataset, you can use the evaluate_on_dataset() method. @@ -94,8 +95,31 @@ class Evaluator(object): "eager execution is enabled.") return control_flow_ops.group([m.init_variables() for _, m in self.metrics]) - def all_metric_results(self): # TODO(josh11b): Add optional summary_writer. - """Returns dict mapping metric name -> value.""" + def all_metric_results(self, summary_logdir=None): + """Computes results for all contained metrics. + + Args: + summary_logdir: An optional string. If specified, metric results + will be written as summaries to this directory. + + Returns: + A `dict` mapping string names to tensors. + """ + if summary_logdir is None: + with summary_ops.never_record_summaries(): + return self._all_metric_results() + else: + def f(): + with summary_ops.create_summary_file_writer( + summary_logdir).as_default(), summary_ops.always_record_summaries(): + return self._all_metric_results() + if context.in_eager_mode(): + return f() + else: + return function.defun(f)() + + def _all_metric_results(self): + """Implementation of `all_metric_results` in the summary context.""" results = {} for name, metric in six.iteritems(self._metrics): results[name] = metric.result() @@ -110,7 +134,9 @@ class Evaluator(object): Args: dataset: Dataset object with the input data to evaluate on. *args: - **kwargs: Optional additional arguments to __call__(). + **kwargs: Optional additional arguments to __call__(), except + `summary_logdir`: if specified, metrics will be written as summaries + to this directory. Returns: @compatibility(eager) @@ -131,17 +157,17 @@ class Evaluator(object): ``` @end_compatibility """ - # TODO(josh11b): Add optional summary_writer. + summary_logdir = kwargs.pop("summary_logdir", None) if context.in_graph_mode(): call_op = self.__call__(dataset.make_one_shot_iterator().get_next(), *args, **kwargs) init_op = self.init_variables() - results_op = self.all_metric_results() + results_op = self.all_metric_results(summary_logdir) return (init_op, call_op, results_op) # Eager case for example in datasets.Iterator(dataset): self.__call__(example, *args, **kwargs) - return self.all_metric_results() + return self.all_metric_results(summary_logdir) @staticmethod def run_evaluation(init_op, call_op, results_op, sess=None): diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py index 4652a69081..02f82cb216 100644 --- a/tensorflow/contrib/eager/python/evaluator_test.py +++ b/tensorflow/contrib/eager/python/evaluator_test.py @@ -18,11 +18,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import tempfile + from tensorflow.contrib.eager.python import evaluator + from tensorflow.contrib.eager.python import metrics +from tensorflow.contrib.summary import summary_test_util from tensorflow.python.data.ops import dataset_ops from tensorflow.python.eager import context from tensorflow.python.eager import test +from tensorflow.python.framework import ops +from tensorflow.python.ops import variables +from tensorflow.python.training import training_util class IdentityModel(object): @@ -71,6 +78,19 @@ class EvaluatorTest(test.TestCase): self.assertEqual(set(["mean"]), set(results.keys())) self.assertEqual(6.0, results["mean"].numpy()) + def testWriteSummaries(self): + e = SimpleEvaluator(IdentityModel()) + e(3.0) + e([5.0, 7.0, 9.0]) + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + + e.all_metric_results(logdir) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 6.0) + def testComposition(self): e = DelegatingEvaluator(PrefixLModel()) e({"inner": 2.0, "outer": 100.0}) @@ -97,7 +117,7 @@ class EvaluatorTest(test.TestCase): self.assertEqual(6.0, results["mean"].numpy()) def testDatasetGraph(self): - with context.graph_mode(), self.test_session(): + with context.graph_mode(), ops.Graph().as_default(), self.test_session(): e = SimpleEvaluator(IdentityModel()) ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) init_op, call_op, results_op = e.evaluate_on_dataset(ds) @@ -105,6 +125,21 @@ class EvaluatorTest(test.TestCase): self.assertEqual(set(["mean"]), set(results.keys())) self.assertEqual(6.0, results["mean"]) + def testWriteSummariesGraph(self): + with context.graph_mode(), ops.Graph().as_default(), self.test_session(): + e = SimpleEvaluator(IdentityModel()) + ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + init_op, call_op, results_op = e.evaluate_on_dataset( + ds, summary_logdir=logdir) + variables.global_variables_initializer().run() + e.run_evaluation(init_op, call_op, results_op) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 6.0) + def testModelProperty(self): m = IdentityModel() e = SimpleEvaluator(m) diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 2df596923b..b945e97a00 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -18,18 +18,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import tempfile from tensorflow.contrib.eager.python import metrics from tensorflow.contrib.summary import summary_ops -from tensorflow.core.util import event_pb2 +from tensorflow.contrib.summary import summary_test_util from tensorflow.python.eager import context from tensorflow.python.eager import test from tensorflow.python.framework import dtypes -from tensorflow.python.lib.io import tf_record from tensorflow.python.ops import array_ops -from tensorflow.python.platform import gfile from tensorflow.python.training import training_util @@ -63,15 +60,9 @@ class MetricsTest(test.TestCase): name="t0").as_default(), summary_ops.always_record_summaries(): m.result() # As a side-effect will write summaries. - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list( - tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].simple_value, 37.0) + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 37.0) def testWeightedMean(self): m = metrics.Mean() diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py index 89031caadc..ca82ea094c 100644 --- a/tensorflow/contrib/summary/summary.py +++ b/tensorflow/contrib/summary/summary.py @@ -29,6 +29,7 @@ from tensorflow.contrib.summary.summary_ops import all_summary_ops from tensorflow.contrib.summary.summary_ops import always_record_summaries from tensorflow.contrib.summary.summary_ops import audio from tensorflow.contrib.summary.summary_ops import create_summary_file_writer +from tensorflow.contrib.summary.summary_ops import eval_dir from tensorflow.contrib.summary.summary_ops import generic from tensorflow.contrib.summary.summary_ops import histogram from tensorflow.contrib.summary.summary_ops import image diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index 9c71bf7740..6028360732 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -19,6 +19,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os + from tensorflow.contrib.summary import gen_summary_ops from tensorflow.python.eager import context from tensorflow.python.framework import constant_op @@ -272,3 +274,8 @@ def audio(name, tensor, sample_rate, max_outputs, family=None): name=scope) return summary_writer_function(name, tensor, function, family=family) + + +def eval_dir(model_dir, name=None): + """Construct a logdir for an eval summary writer.""" + return os.path.join(model_dir, "eval" if not name else "eval_" + name) -- GitLab From 309e340619ab922f1ecb51b8f142283e09bda07d Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Mon, 30 Oct 2017 17:05:22 -0700 Subject: [PATCH 546/573] Avoid uncollectable cycles with a separate deleter object for resources. PiperOrigin-RevId: 173972515 --- tensorflow/contrib/eager/python/datasets.py | 9 +-- .../contrib/eager/python/summary_writer.py | 8 +- tensorflow/contrib/summary/summary_ops.py | 5 +- .../resource_variable_ops_test.py | 13 ++- .../kernel_tests/variable_scope_test.py | 8 ++ .../python/ops/resource_variable_ops.py | 81 +++++++++++-------- 6 files changed, 75 insertions(+), 49 deletions(-) diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py index 357e3420d2..98e6983658 100644 --- a/tensorflow/contrib/eager/python/datasets.py +++ b/tensorflow/contrib/eager/python/datasets.py @@ -80,14 +80,11 @@ class Iterator(object): output_types=self._flat_output_types, output_shapes=self._flat_output_shapes) gen_dataset_ops.make_iterator(ds_variant, self._resource) + # Delete the resource when this object is deleted + self._resource_deleter = resource_variable_ops.EagerResourceDeleter( + handle=self._resource, handle_device="/device:CPU:0") self._device = context.context().device_name - def __del__(self): - if self._resource is not None: - with ops.device("/device:CPU:0"), context.eager_mode(): - resource_variable_ops.destroy_resource_op(self._resource) - self._resource = None - def __iter__(self): return self diff --git a/tensorflow/contrib/eager/python/summary_writer.py b/tensorflow/contrib/eager/python/summary_writer.py index 5a698b92c6..5d8c41b545 100644 --- a/tensorflow/contrib/eager/python/summary_writer.py +++ b/tensorflow/contrib/eager/python/summary_writer.py @@ -114,11 +114,9 @@ class SummaryWriter(object): self._resource = gen_summary_ops.summary_writer(shared_name=self._name) gen_summary_ops.create_summary_file_writer( self._resource, logdir, max_queue, flush_secs, filename_suffix) - - def __del__(self): - if self._resource: - resource_variable_ops.destroy_resource_op(self._resource) - self._resource = None + # Delete the resource when this object is deleted + self._resource_deleter = resource_variable_ops.EagerResourceDeleter( + handle=self._resource, handle_device=self._CPU_DEVICE) def step(self): """Increment the global step counter of this SummaryWriter instance.""" diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index 6028360732..1d1c88944a 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -92,10 +92,9 @@ class SummaryWriter(object): def __init__(self, resource): self._resource = resource - - def __del__(self): if context.in_eager_mode(): - resource_variable_ops.destroy_resource_op(self._resource) + self._resource_deleter = resource_variable_ops.EagerResourceDeleter( + handle=self._resource, handle_device="cpu:0") def set_as_default(self): context.context().summary_writer_resource = self._resource diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 7922e3838f..8f328cea63 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -17,6 +17,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gc + import numpy as np from tensorflow.python.eager import context @@ -38,6 +40,12 @@ from tensorflow.python.platform import test class ResourceVariableOpsTest(test_util.TensorFlowTestCase): + def tearDown(self): + gc.collect() + # This will only contain uncollectable garbage, i.e. reference cycles + # involving objects with __del__ defined. + self.assertEqual(0, len(gc.garbage)) + def testHandleDtypeShapeMatch(self): with self.test_session(): handle = resource_variable_ops.var_handle_op(dtype=dtypes.int32, shape=[]) @@ -477,10 +485,11 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with context.eager_mode(): var = resource_variable_ops.ResourceVariable(initial_value=1.0, name="var8") - var.__del__() + var_handle = var._handle + del var with self.assertRaisesRegexp(errors.NotFoundError, r"Resource .* does not exist."): - resource_variable_ops.destroy_resource_op(var._handle, + resource_variable_ops.destroy_resource_op(var_handle, ignore_lookup_error=False) def testScatterUpdate(self): diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py index efeb25d095..bd4b12b7e8 100644 --- a/tensorflow/python/kernel_tests/variable_scope_test.py +++ b/tensorflow/python/kernel_tests/variable_scope_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gc + import numpy from tensorflow.python.eager import context @@ -39,6 +41,12 @@ from tensorflow.python.platform import test class VariableScopeTest(test.TestCase): + def tearDown(self): + gc.collect() + # This will only contain uncollectable garbage, i.e. reference cycles + # involving objects with __del__ defined. + self.assertEqual(0, len(gc.garbage)) + def testGetVar(self): vs = variable_scope._get_default_variable_store() v = vs.get_variable("v", [1]) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index d7fb6767d1..9e5bb4a225 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -77,6 +77,45 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): return handle +class EagerResourceDeleter(object): + """An object which cleans up a resource handle. + + An alternative to defining a __del__ method on an object. The intended use is + that ResourceVariables or other objects with resource handles will maintain a + single reference to this object. When the parent object is collected, this + object will be too. Even if the parent object is part of a reference cycle, + the cycle will be collectable. + """ + + def __init__(self, handle, handle_device): + self._handle = handle + self._handle_device = handle_device + + def __del__(self): + # Resources follow object-identity when executing eagerly, so it is safe to + # delete the resource we have a handle to. Each Graph has a unique container + # name, which prevents resource sharing. + try: + # This resource was created in eager mode. However, this destructor may be + # running in graph mode (especially during unit tests). To clean up + # successfully, we switch back into eager mode temporarily. + with context.eager_mode(): + with ops.device(self._handle_device): + gen_resource_variable_ops.destroy_resource_op( + self._handle, ignore_lookup_error=True) + except TypeError: + # Suppress some exceptions, mainly for the case when we're running on + # module deletion. Things that can go wrong include the context module + # already being unloaded, self._handle._handle_data no longer being + # valid, and so on. Printing warnings in these cases is silly + # (exceptions raised from __del__ are printed as warnings to stderr). + pass # 'NoneType' object is not callable when the handle has been + # partially unloaded. + except AttributeError: + pass # 'NoneType' object has no attribute 'eager_mode' when context has + # been unloaded. Will catch other module unloads as well. + + def shape_safe_assign_variable_handle(handle, shape, value, name=None): """Helper that checks shape compatibility and assigns variable.""" value_tensor = ops.convert_to_tensor(value) @@ -415,6 +454,15 @@ class ResourceVariable(variables.Variable): ops.add_to_collections(collections, self) elif ops.GraphKeys.GLOBAL_STEP in collections: ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, self) + if not self._in_graph_mode: + # After the handle has been created, set up a way to clean it up when + # executing eagerly. We'll hold the only reference to the deleter, so that + # when this object is garbage collected the deleter will be too. This + # means ResourceVariables can be part of reference cycles without those + # cycles being uncollectable, and means that no __del__ will be defined at + # all in graph mode. + self._handle_deleter = EagerResourceDeleter( + handle=self._handle, handle_device=self._handle_device) def _init_from_proto(self, variable_def, import_scope=None): """Initializes from `VariableDef` proto.""" @@ -454,39 +502,6 @@ class ResourceVariable(variables.Variable): self._constraint = None # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py) - def __del__(self): - if not self._in_graph_mode: - # There is only one ResourceVariable object for each underlying resource - # (cached in the Graph's VariableStore when created with get_variable), so - # it is safe to delete the resource we have a handle to. Each Graph has a - # unique container name in Eager, which prevents resource sharing. - # - # The Graph's VariableStore contains strong references to ResourceVariable - # objects created with get_variable, so this destructor will only be - # callled once the Graph is garbage collected for those objects. However, - # explicitly created ResourceVariables (e.g. through tfe.Variable) may be - # collected earlier. - try: - # We have checked that this ResourceVariable was created in Eager - # mode. However, this destructor may be running in graph mode - # (especially during unit tests). To clean up successfully, we switch - # back into Eager temporarily. - with context.eager_mode(): - with ops.device(self._handle_device): - gen_resource_variable_ops.destroy_resource_op( - self._handle, ignore_lookup_error=True) - except TypeError: - # Suppress some exceptions, mainly for the case when we're running on - # module deletion. Things that can go wrong include the context module - # already being unloaded, self._handle._handle_data no longer being - # valid, and so on. Printing warnings in these cases is silly - # (exceptions raised from __del__ are printed as warnings to stderr). - pass # 'NoneType' object is not callable when the handle has been - # partially unloaded. - except AttributeError: - pass # 'NoneType' object has no attribute 'eager_mode' when context has - # been unloaded. Will catch other module unloads as well. - def __nonzero__(self): return self.__bool__() -- GitLab From 542b323e5a8dda887ad9e27bb697a15471447f8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 17:23:51 -0700 Subject: [PATCH 547/573] Register quint16/qint16 for GatherOp. PiperOrigin-RevId: 173974904 --- tensorflow/core/kernels/gather_op.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/kernels/gather_op.cc b/tensorflow/core/kernels/gather_op.cc index e649c54fa8..7088005e73 100644 --- a/tensorflow/core/kernels/gather_op.cc +++ b/tensorflow/core/kernels/gather_op.cc @@ -140,6 +140,8 @@ class GatherOp : public OpKernel { // Registration of the CPU implementations. TF_CALL_ALL_TYPES(REGISTER_GATHER_CPU); TF_CALL_QUANTIZED_TYPES(REGISTER_GATHER_CPU); +TF_CALL_quint16(REGISTER_GATHER_CPU); +TF_CALL_qint16(REGISTER_GATHER_CPU); #undef REGISTER_GATHER_CPU -- GitLab From 187453d61da2fb3e1f30d40962863f6e18c5a78e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 17:27:08 -0700 Subject: [PATCH 548/573] Change momentum optimizer to allow callable learning_rate and momentum parameters. This can be useful for implementing learninge rate decay. PiperOrigin-RevId: 173975321 --- tensorflow/python/training/momentum.py | 19 +++++++++++++++---- tensorflow/python/training/momentum_test.py | 13 +++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py index 7c00e219fd..cf9530d87c 100644 --- a/tensorflow/python/training/momentum.py +++ b/tensorflow/python/training/momentum.py @@ -28,7 +28,7 @@ class MomentumOptimizer(optimizer.Optimizer): """Optimizer that implements the Momentum algorithm. Computes (if `use_nesterov = False`): - + ``` accumulation = momentum * accumulation + gradient variable -= learning_rate * accumulation @@ -58,6 +58,12 @@ class MomentumOptimizer(optimizer.Optimizer): variable(s) passed to the optimizer. Using Nesterov Momentum makes the variable(s) track the values called `theta_t + mu*v_t` in the paper. + @compatibility(eager) + When eager execution is enabled, learning_rate and momentum can each be a + callable that takes no arguments and returns the actual value to use. This + can be useful for changing these values across different invocations of + optimizer functions. + @end_compatibility """ super(MomentumOptimizer, self).__init__(use_locking, name) self._learning_rate = learning_rate @@ -69,10 +75,15 @@ class MomentumOptimizer(optimizer.Optimizer): self._zeros_slot(v, "momentum", self._name) def _prepare(self): - self._learning_rate_tensor = ops.convert_to_tensor(self._learning_rate, + learning_rate = self._learning_rate + if callable(learning_rate): + learning_rate = learning_rate() + self._learning_rate_tensor = ops.convert_to_tensor(learning_rate, name="learning_rate") - self._momentum_tensor = ops.convert_to_tensor(self._momentum, - name="momentum") + momentum = self._momentum + if callable(momentum): + momentum = momentum() + self._momentum_tensor = ops.convert_to_tensor(momentum, name="momentum") def _apply_dense(self, grad, var): mom = self.get_slot(var, "momentum") diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py index d354ea443c..3c8f472d6f 100644 --- a/tensorflow/python/training/momentum_test.py +++ b/tensorflow/python/training/momentum_test.py @@ -44,7 +44,7 @@ class MomentumOptimizerTest(test.TestCase): var = var - accum * lr * momentum return var, accum - def doTestBasic(self, use_resource=False): + def doTestBasic(self, use_resource=False, use_callable_params=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): if use_resource: var0 = resource_variable_ops.ResourceVariable( @@ -56,8 +56,13 @@ class MomentumOptimizerTest(test.TestCase): var1 = variables.Variable([3.0, 4.0], dtype=dtype) grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + learning_rate = lambda: 2.0 + momentum = lambda: 0.9 + if not use_callable_params: + learning_rate = learning_rate() + momentum = momentum() mom_opt = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9) + learning_rate=learning_rate, momentum=momentum) mom_update = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) @@ -125,6 +130,10 @@ class MomentumOptimizerTest(test.TestCase): def testResourceBasic(self): self.doTestBasic(use_resource=True) + def testBasicCallableParams(self): + with context.eager_mode(): + self.doTestBasic(use_resource=True, use_callable_params=True) + def testNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: with self.test_session(): -- GitLab From 2ba52985657e8189a19f1be52448b8268ccd879a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 18:15:11 -0700 Subject: [PATCH 549/573] Initial add of docs for Tensorflow on Mobile. PiperOrigin-RevId: 173980290 --- tensorflow/docs_src/mobile/android_build.md | 176 +++++++ tensorflow/docs_src/mobile/index.md | 238 +++++++++ tensorflow/docs_src/mobile/ios_build.md | 107 ++++ tensorflow/docs_src/mobile/leftnav_files | 8 + tensorflow/docs_src/mobile/linking_libs.md | 243 +++++++++ tensorflow/docs_src/mobile/optimizing.md | 497 +++++++++++++++++++ tensorflow/docs_src/mobile/prepare_models.md | 301 +++++++++++ 7 files changed, 1570 insertions(+) create mode 100644 tensorflow/docs_src/mobile/android_build.md create mode 100644 tensorflow/docs_src/mobile/index.md create mode 100644 tensorflow/docs_src/mobile/ios_build.md create mode 100644 tensorflow/docs_src/mobile/leftnav_files create mode 100644 tensorflow/docs_src/mobile/linking_libs.md create mode 100644 tensorflow/docs_src/mobile/optimizing.md create mode 100644 tensorflow/docs_src/mobile/prepare_models.md diff --git a/tensorflow/docs_src/mobile/android_build.md b/tensorflow/docs_src/mobile/android_build.md new file mode 100644 index 0000000000..030cd0d051 --- /dev/null +++ b/tensorflow/docs_src/mobile/android_build.md @@ -0,0 +1,176 @@ +# Building TensorFlow on Android + +To get you started working with TensorFlow on Android, we'll walk through two +ways to build our TensorFlow mobile demos and deploying them on an Android +device. The first is Android Studio, which lets you build and deploy in an +IDE. The second is building with Bazel and deploying with ADB on the command +line. + +Why choose one or the other of these methods? + +The simplest way to use TensorFlow on Android is to use Android Studio. If you +aren't planning to customize your TensorFlow build at all, or if you want to use +Android Studio's editor and other features to build an app and just want to add +TensorFlow to it, we recommend using Android Studio. + +If you are using custom ops, or have some other reason to build TensorFlow from +scratch, scroll down and see our instructions +for [building the demo with Bazel](#build_the_demo_using_bazel). + +## Build the demo using Android Studio + +**Prerequisites** + +If you haven't already, do the following two things: + +- Install [Android Studio](https://developer.android.com/studio/index.html), + following the instructions on their website. + +- Clone the TensorFlow repository from Github: + + git clone https://github.com/tensorflow/tensorflow + +**Building** + +1. Open Android Studio, and from the Welcome screen, select **Open an existing + Android Studio project**. + +2. From the **Open File or Project** window that appears, navigate to and select + the `tensorflow/examples/android` directory from wherever you cloned the + TensorFlow Github repo. Click OK. + + If it asks you to do a Gradle Sync, click OK. + + You may also need to install various platforms and tools, if you get + errors like "Failed to find target with hash string 'android-23' and similar. + +3. Open the `build.gradle` file (you can go to **1:Project** in the side panel + and find it under the **Gradle Scripts** zippy under **Android**). Look for + the `nativeBuildSystem` variable and set it to `none` if it isn't already: + + // set to 'bazel', 'cmake', 'makefile', 'none' + def nativeBuildSystem = 'none' + +4. Click the Run button (the green arrow) or use **Run -> Run 'android'** from the top menu. + + If it asks you to use Instant Run, click **Proceed Without Instant Run**. + + Also, you need to have an Android device plugged in with developer options + enabled at this + point. See [here](https://developer.android.com/studio/run/device.html) for + more details on setting up developer devices. + +This installs three apps on your phone that are all part of the TensorFlow +Demo. See [Android Sample Apps](#android_sample_apps) for more information about +them. + +## Adding TensorFlow to your apps using Android Studio + +To add TensorFlow to your own apps on Android, the simplest way is to add the +following lines to your Gradle build file: + + allprojects { + repositories { + jcenter() + } + } + + dependencies { + compile 'org.tensorflow:tensorflow-android:+' + } + +This automatically downloads the latest stable version of TensorFlow as an AAR +and installs it in your project. + +## Build the demo using Bazel + +Another way to use TensorFlow on Android is to build an APK +using [Bazel](https://bazel.build/) and load it onto your device +using [ADB](https://developer.android.com/studio/command-line/adb.html). This +requires some knowledge of build systems and Android developer tools, but we'll +guide you through the basics here. + +- First, follow our instructions for @{$install/install_sources$installing from + sources}. This will also guide you through installing Bazel and cloning the + TensorFlow code. + +- Download the Android [SDK](https://developer.android.com/studio/index.html) + and [NDK](https://developer.android.com/ndk/downloads/index.html) if you do + not already have them. You need at least version 12b of the NDK, and 23 of the + SDK. + +- In your copy of the TensorFlow source, update the + [WORKSPACE](https://github.com/tensorflow/tensorflow/blob/master/WORKSPACE) + file with the location of your SDK and NDK, where it says <PATH_TO_NDK> + and <PATH_TO_SDK>. + +- Run Bazel to build the demo APK: + + bazel build -c opt //tensorflow/examples/android:tensorflow_demo + +- Use [ADB](https://developer.android.com/studio/command-line/adb.html#move) to + install the APK onto your device: + + adb install -r bazel-bin/tensorflow/examples/android/tensorflow_demo.apk + +Note: In general when compiling for Android with Bazel you need +`--config=android` on the Bazel command line, though in this case this +particular example is Android-only, so you don't need it here. + +This installs three apps on your phone that are all part of the TensorFlow +Demo. See [Android Sample Apps](#android_sample_apps) for more information about +them. + +## Android Sample Apps + +The +[Android example code](https://www.tensorflow.org/code/tensorflow/examples/android/) is +a single project that builds and installs three sample apps which all use the +same underlying code. The sample apps all take video input from a phone's +camera: + +- **TF Classify** uses the Inception v3 model to label the objects it’s pointed + at with classes from Imagenet. There are only 1,000 categories in Imagenet, + which misses most everyday objects and includes many things you’re unlikely to + encounter often in real life, so the results can often be quite amusing. For + example there’s no ‘person’ category, so instead it will often guess things it + does know that are often associated with pictures of people, like a seat belt + or an oxygen mask. If you do want to customize this example to recognize + objects you care about, you can use + the + [TensorFlow for Poets codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) as + an example for how to train a model based on your own data. + +- **TF Detect** uses a multibox model to try to draw bounding boxes around the + locations of people in the camera. These boxes are annotated with the + confidence for each detection result. Results will not be perfect, as this + kind of object detection is still an active research topic. The demo also + includes optical tracking for when objects move between frames, which runs + more frequently than the TensorFlow inference. This improves the user + experience since the apparent frame rate is faster, but it also gives the + ability to estimate which boxes refer to the same object between frames, which + is important for counting objects over time. + +- **TF Stylize** implements a real-time style transfer algorithm on the camera + feed. You can select which styles to use and mix between them using the + palette at the bottom of the screen, and also switch out the resolution of the + processing to go higher or lower rez. + +When you build and install the demo, you'll see three app icons on your phone, +one for each of the demos. Tapping on them should open up the app and let you +explore what they do. You can enable profiling statistics on-screen by tapping +the volume up button while they’re running. + +### Android Inference Library + +Because Android apps need to be written in Java, and core TensorFlow is in C++, +TensorFlow has a JNI library to interface between the two. Its interface is aimed +only at inference, so it provides the ability to load a graph, set up inputs, +and run the model to calculate particular outputs. You can see the full +documentation for the minimal set of methods in +[TensorFlowInferenceInterface.java](https://www.tensorflow.org/code/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java) + +The demos applications use this interface, so they’re a good place to look for +example usage. You can download prebuilt binary jars +at +[ci.tensorflow.org](https://ci.tensorflow.org/view/Nightly/job/nightly-android/). diff --git a/tensorflow/docs_src/mobile/index.md b/tensorflow/docs_src/mobile/index.md new file mode 100644 index 0000000000..a6f1422f6f --- /dev/null +++ b/tensorflow/docs_src/mobile/index.md @@ -0,0 +1,238 @@ +# Building Mobile Apps with TensorFlow + +TensorFlow was designed from the ground up to be a good deep learning solution +for mobile platforms like Android and iOS. This guide is to help you understand +how to integrate TensorFlow into your mobile apps effectively and efficiently. + +## About this Guide + +This guide is aimed at developers who have a TensorFlow model that’s +successfully working in a desktop environment, and who want to integrate it into +a mobile application. Here are the main challenges you’ll face during that +process: + +- Understanding how to use Tensorflow for mobile. +- Building TensorFlow for your platform. +- Integrating the TensorFlow library into your application. +- Preparing your model file for mobile deployment. +- Optimizing for latency, RAM usage, model file size, and binary size. + +## Why run TensorFlow on mobile? + +Traditionally, deep learning has been associated with data centers and giant +clusters of high-powered GPU machines. However, it can be very expensive and +time-consuming to send all of the data a device has access to across a network +connection. Running on mobile makes it possible to deliver very interactive +applications in a way that’s not possible when you have to wait for a network +round trip. + +Here are some common use cases for on-device deep learning: + +### Speech Recognition + +There are a lot of interesting applications that can be built with a +speech-driven interface, and many of these require on-device processing. Most of +the time a user isn’t giving commands, and so streaming audio continuously to a +remote server would be a waste of bandwidth, since it would mostly be silence or +background noises. To solve this problem it’s common to have a small neural +network running on-device @{$tutorials/audio_recognition$listening out for a +particular keyword}. Once that keyword has been spotted, the rest of the +conversation can be transmitted over to the server for further processing if +more computing power is needed. + +### Image Recognition + +It can be very useful for a mobile app to be able to make sense of a camera +image. If your users are taking photos, recognizing what’s in them can help your +camera apps apply appropriate filters, or label the photos so they’re easily +findable. It’s important for embedded applications too, since you can use image +sensors to detect all sorts of interesting conditions, whether it’s spotting +endangered animals in the wild +or +[reporting how late your train is running](https://svds.com/tensorflow-image-recognition-raspberry-pi/). + +TensorFlow comes with several examples of recognizing the types of objects +inside images along with a variety of different pre-trained models, and they can +all be run on mobile devices. You can try out +our +[Tensorflow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) and +[Tensorflow for Poets 2: Optimize for Mobile](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/index.html#0) codelabs to +see how to take a pretrained model and run some very fast and lightweight +training to teach it to recognize specific objects, and then optimize it to +run on mobile. + +### Object Localization + +Sometimes it’s important to know where objects are in an image as well as what +they are. There are lots of augmented reality use cases that could benefit a +mobile app, such as guiding users to the right component when offering them +help fixing their wireless network or providing informative overlays on top of +landscape features. Embedded applications often need to count objects that are +passing by them, whether it’s pests in a field of crops, or people, cars and +bikes going past a street lamp. + +TensorFlow offers a pretrained model for drawing bounding boxes around people +detected in images, together with tracking code to follow them over time. The +tracking is especially important for applications where you’re trying to count +how many objects are present over time, since it gives you a good idea when a +new object enters or leaves the scene. We have some sample code for this +available for Android [on +Github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android), +and also a [more general object detection +model](https://github.com/tensorflow/models/tree/master/object_detection/README.md) +available as well. + +### Gesture Recognition + +It can be useful to be able to control applications with hand or other +gestures, either recognized from images or through analyzing accelerometer +sensor data. Creating those models is beyond the scope of this guide, but +TensorFlow is an effective way of deploying them. + +### Optical Character Recognition + +Google Translate’s live camera view is a great example of how effective +interactive on-device detection of text can be. + +
+ +
+ +There are multiple steps involved in recognizing text in images. You first have +to identify the areas where the text is present, which is a variation on the +object localization problem, and can be solved with similar techniques. Once you +have an area of text, you then need to interpret it as letters, and then use a +language model to help guess what words they represent. The simplest way to +estimate what letters are present is to segment the line of text into individual +letters, and then apply a simple neural network to the bounding box of each. You +can get good results with the kind of models used for MNIST, which you can find +in TensorFlow’s tutorials, though you may want a higher-resolution input. A +more advanced alternative is to use an LSTM model to process a whole line of +text at once, with the model itself handling the segmentation into different +characters. + +### Translation + +Translating from one language to another quickly and accurately, even if you +don’t have a network connection, is an important use case. Deep networks are +very effective at this sort of task, and you can find descriptions of a lot of +different models in the literature. Often these are sequence-to-sequence +recurrent models where you’re able to run a single graph to do the whole +translation, without needing to run separate parsing stages. + +### Text Classification + +If you want to suggest relevant prompts to users based on what they’re typing or +reading, it can be very useful to understand the meaning of the text. This is +where text classification comes in. Text classification is an umbrella term +that covers everything from sentiment analysis to topic discovery. You’re likely +to have your own categories or labels that you want to apply, so the best place +to start is with an example +like +[Skip-Thoughts](https://github.com/tensorflow/models/tree/master/skip_thoughts/), +and then train on your own examples. + +### Voice Synthesis + +A synthesized voice can be a great way of giving users feedback or aiding +accessibility, and recent advances such as +[WaveNet](https://deepmind.com/blog/wavenet-generative-model-raw-audio/) show +that deep learning can offer very natural-sounding speech. + +## How does it fit with the cloud? + +These examples of use cases give an idea of how on-device networks can +complement cloud services. Cloud has a great deal of computing power in a +controlled environment, but running on devices can offer higher interactivity. +In situations where the cloud is unavailable, or your cloud capacity is limited, +you can provide an offline experience, or reduce cloud workload by processing +easy cases on device. + +Doing on-device computation can also signal when it's time to switch to working +on the cloud. A good example of this is hotword detection in speech. Since +devices are able to constantly listen out for the keywords, this then triggers a +lot of traffic to cloud-based speech recognition once one is recognised. Without +the on-device component, the whole application wouldn’t be feasible, and this +pattern exists across several other applications as well. Recognizing that some +sensor input is interesting enough for further processing makes a lot of +interesting products possible. + +## What hardware and software should you have? + +TensorFlow runs on Ubuntu Linux, Windows 10, and OS X. For a list of all +supported operating systems and instructions to install TensorFlow, see +@{$install$Installing Tensorflow}. + +Some of the scripts in this guide require you to compile TensorFlow from source, +so you’ll need more than just `pip install` to work through all the sample code. + +To try out the mobile examples, you’ll need a device set up for development, +using +either [Android Studio](https://developer.android.com/studio/install.html), +or [XCode](https://developer.apple.com/xcode/) if you're developing for iOS. + +## What should you do before you get started? + +Before thinking about how to get your solution on mobile: + +1. Determine whether your problem is solvable by mobile machine learning +2. Create a labelled dataset to define your problem +3. Pick an effective model for the problem + +We'll discuss these in more detail below. + +### Is your problem solvable by mobile machine learning? + +Once you have an idea of the problem you want to solve, you need to make a plan +of how to build your solution. The most important first step is making sure that +your problem is actually solvable, and the best way to do that is to mock it up +using humans in the loop. + +For example, if you want to drive a robot toy car using voice commands, try +recording some audio from the device and listen back to it to see if you can +make sense of what’s being said. Often you’ll find there are problems in the +capture process, such as the motor drowning out speech or not being able to hear +at a distance, and you should tackle these problems before investing in the +modeling process. + +Another example would be giving photos taken from your app to people see if they +can classify what’s in them, in the way you’re looking for. If they can’t do +that (for example, trying to estimate calories in food from photos may be +impossible because all white soups look the same), then you’ll need to redesign +your experience to cope with that. A good rule of thumb is that if a human can’t +handle the task then it will be difficult to train a computer to do better. + +### Create a labelled dataset + +After you’ve solved any fundamental issues with your use case, you need to +create a labeled dataset to define what problem you’re trying to solve. This +step is extremely important, moreso than picking which model to use. You want it +to be as representative as possible of your actual use case, since the model +will only be effective at the task you teach it. It’s also worth investing in +tools to make labeling the data as efficient and accurate as possible. For +example, if you’re able to switch from having to click a button on a web +interface to simple keyboard shortcuts, you may be able to speed up the +generation process a lot. You should also start by doing the initial labeling +yourself, so you can learn about the difficulties and likely errors, and +possibly change your labeling or data capture process to avoid them. Once you +and your team are able to consistently label examples (that is once you +generally agree on the same labels for most examples), you can then try and +capture your knowledge in a manual and teach external raters how to run the same +process. + +### Pick an effective model + +The next step is to pick an effective model to use. You might be able to avoid +training a model from scratch if someone else has already implemented a model +similar to what you need; we have a repository of models implemented in +TensorFlow [on Github](https://github.com/tensorflow/models) that you can look +through. Lean towards the simplest model you can find, and try to get started as +soon as you have even a small amount of labelled data, since you’ll get the best +results when you’re able to iterate quickly. The shorter the time it takes to +try training a model and running it in s real application, the better overall +results you’ll see. It’s common for an algorithm to get great training accuracy +numbers but then fail to be useful within a real application because there’s a +mismatch between the dataset and real usage. Prototype end-to-end usage as soon +as possible to create a consistent user experience. diff --git a/tensorflow/docs_src/mobile/ios_build.md b/tensorflow/docs_src/mobile/ios_build.md new file mode 100644 index 0000000000..2e6d3bf90e --- /dev/null +++ b/tensorflow/docs_src/mobile/ios_build.md @@ -0,0 +1,107 @@ +# Building TensorFlow on iOS + +## Using CocoaPods + +The simplest way to get started with TensorFlow on iOS is using the CocoaPods +package management system. You can add the `TensorFlow-experimental` pod to your +Podfile, which installs a universal binary framework. This makes it easy to get +started but has the disadvantage of being hard to customize, which is important +in case you want to shrink your binary size. If you do need the ability to +customize your libraries, see later sections on how to do that. + +## Creating your own app + +If you'd like to add TensorFlow capabilities to your own app, do the following: + +- Create your own app or load your already-created app in XCode. + +- Add a file named Podfile at the project root directory with the following content: + + target 'YourProjectName' + pod 'TensorFlow-experimental' + +- Run `pod install` to download and install the `TensorFlow-experimental` pod. + +- Open `YourProjectName.xcworkspace` and add your code. + +- In your app's **Build Settings**, make sure to add `$(inherited)` to the + **Other Linker Flags**, and **Header Search Paths** sections. + +## Running the Samples + +You'll need Xcode 7.3 or later to run our iOS samples. + +There are currently three examples: simple, benchmark, and camera. For now, you +can download the sample code by cloning the main tensorflow repository (we are +planning to make the samples available as a separate repository later). + +From the root of the tensorflow folder, download [Inception +v1](https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip), +and extract the label and graph files into the data folders inside both the +simple and camera examples using these steps: + + mkdir -p ~/graphs + curl -o ~/graphs/inception5h.zip \ + https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \ + && unzip ~/graphs/inception5h.zip -d ~/graphs/inception5h + cp ~/graphs/inception5h/* tensorflow/examples/ios/benchmark/data/ + cp ~/graphs/inception5h/* tensorflow/examples/ios/camera/data/ + cp ~/graphs/inception5h/* tensorflow/examples/ios/simple/data/ + +Change into one of the sample directories, download the +[Tensorflow-experimental](https://cocoapods.org/pods/TensorFlow-experimental) +pod, and open the Xcode workspace. Note that installing the pod can take a long +time since it is big (~450MB). If you want to run the simple example, then: + + cd tensorflow/examples/ios/simple + pod install + open tf_simple_example.xcworkspace # note .xcworkspace, not .xcodeproj + # this is created by pod install + +Run the simple app in the XCode simulator. You should see a single-screen app +with a **Run Model** button. Tap that, and you should see some debug output +appear below indicating that the example Grace Hopper image in directory data +has been analyzed, with a military uniform recognized. + +Run the other samples using the same process. The camera example requires a real +device connected. Once you build and run that, you should get a live camera view +that you can point at objects to get real-time recognition results. + +### iOS Example details + +There are three demo applications for iOS, all defined in Xcode projects inside +[tensorflow/examples/ios](https://www.tensorflow.org/code/tensorflow/examples/ios/). + +- **Simple**: This is a minimal example showing how to load and run a TensorFlow + model in as few lines as possible. It just consists of a single view with a + button that executes the model loading and inference when its pressed. + +- **Camera**: This is very similar to the Android TF Classify demo. It loads + Inception v3 and outputs its best label estimate for what’s in the live camera + view. As with the Android version, you can train your own custom model using + TensorFlow for Poets and drop it into this example with minimal code changes. + +- **Benchmark**: is quite close to Simple, but it runs the graph repeatedly and + outputs similar statistics to the benchmark tool on Android. + + +### Troubleshooting + +- Make sure you use the TensorFlow-experimental pod (and not TensorFlow). + +- The TensorFlow-experimental pod is current about ~450MB. The reason it is so + big is because we are bundling multiple platforms, and the pod includes all + TensorFlow functionality (e.g. operations). The final app size after build is + substantially smaller though (~25MB). Working with the complete pod is + convenient during development, but see below section on how you can build your + own custom TensorFlow library to reduce the size. + +## Building the TensorFlow iOS libraries from source + +While Cocapods is the quickest and easiest way of getting started, you sometimes +need more flexibility to determine which parts of TensorFlow your app should be +shipped with. For such cases, you can build the iOS libraries from the +sources. [This +guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/ios#building-the-tensorflow-ios-libraries-from-source) +contains detailed instructions on how to do that. + diff --git a/tensorflow/docs_src/mobile/leftnav_files b/tensorflow/docs_src/mobile/leftnav_files new file mode 100644 index 0000000000..347c07d233 --- /dev/null +++ b/tensorflow/docs_src/mobile/leftnav_files @@ -0,0 +1,8 @@ +### TensorFlow for Mobile +index.md +android_build.md +ios_build.md +#raspi_build.md until this section gets rewritten, or TFLite takes over +linking_libs.md +prepare_models.md +optimizing.md diff --git a/tensorflow/docs_src/mobile/linking_libs.md b/tensorflow/docs_src/mobile/linking_libs.md new file mode 100644 index 0000000000..2a0a77c92d --- /dev/null +++ b/tensorflow/docs_src/mobile/linking_libs.md @@ -0,0 +1,243 @@ +# Integrating TensorFlow libraries + +Once you have made some progress on a model that addresses the problem you’re +trying to solve, it’s important to test it out inside your application +immediately. There are often unexpected differences between your training data +and what users actually encounter in the real world, and getting a clear picture +of the gap as soon as possible improves the product experience. + +This page talks about how to integrate the TensorFlow libraries into your own +mobile applications, once you have already successfully built and deployed the +TensorFlow mobile demo apps. + +## Linking the library + +After you've managed to build the examples, you'll probably want to call +TensorFlow from one of your existing applications. The very easiest way to do +this is to use the Pod installation steps described +@{$mobile/ios_build#using_cocoapods$here}, but if you want to build TensorFlow +from source (for example to customize which operators are included) you'll need +to break out TensorFlow as a framework, include the right header files, and link +against the built libraries and dependencies. + +### Android + +For Android, you just need to link in a Java library contained in a JAR file +called `libandroid_tensorflow_inference_java.jar`. There are three ways to +include this functionality in your program: + +1. Include the jcenter AAR which contains it, as in this + [example app](https://github.com/googlecodelabs/tensorflow-for-poets-2/blob/master/android/build.gradle#L59-L65) + +2. Download the nightly precompiled version from +[ci.tensorflow.org](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/). + +3. Build the JAR file yourself using the instructions [in our Android Github repo](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/android) + +### iOS + +Pulling in the TensorFlow libraries on iOS is a little more complicated. Here is +a checklist of what you’ll need to do to your iOS app: + +- Link against tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a, usually + by adding `-L/your/path/tensorflow/contrib/makefile/gen/lib/` and + `-ltensorflow-core` to your linker flags. + +- Link against the generated protobuf libraries by adding + `-L/your/path/tensorflow/contrib/makefile/gen/protobuf_ios/lib` and + `-lprotobuf` and `-lprotobuf-lite` to your command line. + +- For the include paths, you need the root of your TensorFlow source folder as + the first entry, followed by + `tensorflow/contrib/makefile/downloads/protobuf/src`, + `tensorflow/contrib/makefile/downloads`, + `tensorflow/contrib/makefile/downloads/eigen`, and + `tensorflow/contrib/makefile/gen/proto`. + +- Make sure your binary is built with `-force_load` (or the equivalent on your + platform), aimed at the TensorFlow library to ensure that it’s linked + correctly. More detail on why this is necessary can be found in the next + section, [Global constructor magic](#global_constructor_magic). On Linux-like + platforms, you’ll need different flags, more like + `-Wl,--allow-multiple-definition -Wl,--whole-archive`. + +You’ll also need to link in the Accelerator framework, since this is used to +speed up some of the operations. + +## Global constructor magic + +One of the subtlest problems you may run up against is the “No session factory +registered for the given session options” error when trying to call TensorFlow +from your own application. To understand why this is happening and how to fix +it, you need to know a bit about the architecture of TensorFlow. + +The framework is designed to be very modular, with a thin core and a large +number of specific objects that are independent and can be mixed and matched as +needed. To enable this, the coding pattern in C++ had to let modules easily +notify the framework about the services they offer, without requiring a central +list that has to be updated separately from each implementation. It also had to +allow separate libraries to add their own implementations without needing a +recompile of the core. + +To achieve this capability, TensorFlow uses a registration pattern in a lot of +places. In the code, it looks like this: + + class MulKernel : OpKernel { + Status Compute(OpKernelContext* context) { … } + }; + REGISTER_KERNEL(MulKernel, “Mul”); + +This would be in a standalone `.cc` file linked into your application, either +as part of the main set of kernels or as a separate custom library. The magic +part is that the `REGISTER_KERNEL()` macro is able to inform the core of +TensorFlow that it has an implementation of the Mul operation, so that it can be +called in any graphs that require it. + +From a programming point of view, this setup is very convenient. The +implementation and registration code live in the same file, and adding new +implementations is as simple as compiling and linking it in. The difficult part +comes from the way that the `REGISTER_KERNEL()` macro is implemented. C++ +doesn’t offer a good mechanism for doing this sort of registration, so we have +to resort to some tricky code. Under the hood, the macro is implemented so that +it produces something like this: + + class RegisterMul { + public: + RegisterMul() { + global_kernel_registry()->Register(“Mul”, [](){ + return new MulKernel() + }); + } + }; + RegisterMul g_register_mul; + +This sets up a class `RegisterMul` with a constructor that tells the global +kernel registry what function to call when somebody asks it how to create a +“Mul” kernel. Then there’s a global object of that class, and so the constructor +should be called at the start of any program. + +While this may sound sensible, the unfortunate part is that the global object +that’s defined is not used by any other code, so linkers not designed with this +in mind will decide that it can be deleted. As a result, the constructor is +never called, and the class is never registered. All sorts of modules use this +pattern in TensorFlow, and it happens that `Session` implementations are the +first to be looked for when the code is run, which is why it shows up as the +characteristic error when this problem occurs. + +The solution is to force the linker to not strip any code from the library, even +if it believes it’s unused. On iOS, this step can be accomplished with the +`-force_load` flag, specifying a library path, and on Linux you need +`--whole-archive`. These persuade the linker to not be as aggressive about +stripping, and should retain the globals. + +The actual implementation of the various `REGISTER_*` macros is a bit more +complicated in practice, but they all suffer the same underlying problem. If +you’re interested in how they work, [op_kernel.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_kernel.h#L1091) +is a good place to start investigating. + +## Protobuf problems + +TensorFlow relies on +the [Protocol Buffer](https://developers.google.com/protocol-buffers/) library, +commonly known as protobuf. This library takes definitions of data structures +and produces serialization and access code for them in a variety of +languages. The tricky part is that this generated code needs to be linked +against shared libraries for the exact same version of the framework that was +used for the generator. This can be an issue when `protoc`, the tool used to +generate the code, is from a different version of protobuf than the libraries in +the standard linking and include paths. For example, you might be using a copy +of `protoc` that was built locally in `~/projects/protobuf-3.0.1.a`, but you have +libraries installed at `/usr/local/lib` and `/usr/local/include` that are from +3.0.0. + +The symptoms of this issue are errors during the compilation or linking phases +with protobufs. Usually, the build tools take care of this, but if you’re using +the makefile, make sure you’re building the protobuf library locally and using +it, as shown in [this Makefile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/Makefile#L18). + +Another situation that can cause problems is when protobuf headers and source +files need to be generated as part of the build process. This process makes +building more complex, since the first phase has to be a pass over the protobuf +definitions to create all the needed code files, and only after that can you go +ahead and do a build of the library code. + +### Multiple versions of protobufs in the same app + +Protobufs generate headers that are needed as part of the C++ interface to the +overall TensorFlow library. This complicates using the library as a standalone +framework. + +If your application is already using version 1 of the protocol buffers library, +you may have trouble integrating TensorFlow because it requires version 2. If +you just try to link both versions into the same binary, you’ll see linking +errors because some of the symbols clash. To solve this particular problem, we +have an experimental script at [rename_protobuf.sh](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/rename_protobuf.sh). + +You need to run this as part of the makefile build, after you’ve downloaded all +the dependencies: + + tensorflow/contrib/makefile/download_dependencies.sh + tensorflow/contrib/makefile/rename_protobuf.sh + +## Calling the TensorFlow API + +Once you have the framework available, you then need to call into it. The usual +pattern is that you first load your model, which represents a preset set of +numeric computations, and then you run inputs through that model (for example, +images from a camera) and receive outputs (for example, predicted labels). + +On Android, we provide the Java Inference Library that is focused on just this +use case, while on iOS and Raspberry Pi you call directly into the C++ API. + +### Android + +Here’s what a typical Inference Library sequence looks like on Android: + + // Load the model from disk. + TensorFlowInferenceInterface inferenceInterface = + new TensorFlowInferenceInterface(assetManager, modelFilename); + + // Copy the input data into TensorFlow. + inferenceInterface.feed(inputName, floatValues, 1, inputSize, inputSize, 3); + + // Run the inference call. + inferenceInterface.run(outputNames, logStats); + + // Copy the output Tensor back into the output array. + inferenceInterface.fetch(outputName, outputs); + +You can find the source of this code in the [Android examples](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java#L107). + +### iOS and Raspberry Pi + +Here’s the equivalent code for iOS and Raspberry Pi: + + // Load the model. + PortableReadFileToProto(file_path, &tensorflow_graph); + + // Create a session from the model. + tensorflow::Status s = session->Create(tensorflow_graph); + if (!s.ok()) { + LOG(FATAL) << "Could not create TensorFlow Graph: " << s; + } + + // Run the model. + std::string input_layer = "input"; + std::string output_layer = "output"; + std::vector outputs; + tensorflow::Status run_status = session->Run({{input_layer, image_tensor}}, + {output_layer}, {}, &outputs); + if (!run_status.ok()) { + LOG(FATAL) << "Running model failed: " << run_status; + } + + // Access the output data. + tensorflow::Tensor* output = &outputs[0]; + +This is all based on the +[iOS sample code](https://www.tensorflow.org/code/tensorflow/examples/ios/simple/RunModelViewController.mm), +but there’s nothing iOS-specific; the same code should be usable on any platform +that supports C++. + +You can also find specific examples for Raspberry Pi +[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/pi_examples/label_image/label_image.cc). diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md new file mode 100644 index 0000000000..1da8be5689 --- /dev/null +++ b/tensorflow/docs_src/mobile/optimizing.md @@ -0,0 +1,497 @@ +# Optimizing for mobile + +There are some special issues that you have to deal with when you’re trying to +ship on mobile or embedded devices, and you’ll need to think about these as +you’re developing your model. + +These issues are: + +- Model and Binary Size +- App speed and model loading speed +- Performance and threading + +We'll discuss a few of these below. + +## What are the minimum device requirements for TensorFlow? + +You need at least one megabyte of program memory and several megabytes of RAM to +run the base TensorFlow runtime, so it’s not suitable for DSPs or +microcontrollers. Other than those, the biggest constraint is usually the +calculation speed of the device, and whether you can run the model you need for +your application with a low enough latency. You can use the benchmarking tools +in [How to Profile your Model](#how_to_profile_your_model) to get an idea of how +many FLOPs are required for a model, and then use that to make rule-of-thumb +estimates of how fast they will run on different devices. For example, a modern +smartphone might be able to run 10 GFLOPs per second, so the best you could hope +for from a 5 GFLOP model is two frames per second, though you may do worse +depending on what the exact computation patterns are. + +This model dependence means that it’s possible to run TensorFlow even on very +old or constrained phones, as long as you optimize your network to fit within +the latency budget and possibly within limited RAM too. For memory usage, you +mostly need to make sure that the intermediate buffers that TensorFlow creates +aren’t too large, which you can examine in the benchmark output too. + +## Speed + +One of the highest priorities of most model deployments is figuring out how to +run the inference fast enough to give a good user experience. The first place to +start is by looking at the total number of floating point operations that are +required to execute the graph. You can get a very rough estimate of this by +using the `benchmark_model` tool: + + bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \ + bazel-bin/tensorflow/tools/benchmark/benchmark_model \ + --graph=/tmp/inception_graph.pb --input_layer="Mul:0" \ + --input_layer_shape="1,299,299,3" --input_layer_type="float" \ + --output_layer="softmax:0" --show_run_order=false --show_time=false \ + --show_memory=false --show_summary=true --show_flops=true --logtostderr + +This should show you an estimate of how many operations are needed to run the +graph. You can then use that information to figure out how feasible your model +is to run on the devices you’re targeting. For an example, a high-end phone from +2016 might be able to do 20 billion FLOPs per second, so the best speed you +could hope for from a model that requires 10 billion FLOPs is around 500ms. On a +device like the Raspberry Pi 3 that can do about 5 billion FLOPs, you may only +get one inference every two seconds. + +Having this estimate helps you plan for what you’ll be able to realistically +achieve on a device. If the model is using too many ops, then there are a lot of +opportunities to optimize the architecture to reduce that number. + +Advanced techniques include [SqueezeNet](https://arxiv.org/abs/1602.07360) +and [MobileNet](https://arxiv.org/abs/1704.04861), which are architectures +designed to produce models for mobile -- lean and fast but with a small accuracy +cost. You can also just look at alternative models, even older ones, which may +be smaller. For example, Inception v1 only has around 7 million parameters, +compared to Inception v3’s 24 million, and requires only 3 billion FLOPs rather +than 9 billion for v3. + +## Model Size + +Models that run on a device need to be stored somewhere on the device, and very +large neural networks can be hundreds of megabytes. Most users are reluctant to +download very large app bundles from app stores, so you want to make your model +as small as possible. Furthermore, smaller neural networks can persist in and +out of a mobile device's memory faster. + +To understand how large your network will be on disk, start by looking at the +size on disk of your `GraphDef` file after you’ve run `freeze_graph` and +`strip_unused_nodes` on it (see @{$mobile/prepare_models$Preparing models} for +more details on these tools), since then it should only contain +inference-related nodes. To double-check that your results are as expected, run +the `summarize_graph` tool to see how many parameters are in constants: + + bazel build tensorflow/tools/graph_transforms:summarize_graph && \ + bazel-bin/tensorflow/tools/graph_transforms/summarize_graph \ + --in_graph=/tmp/tensorflow_inception_graph.pb + +That command should give you output that looks something like this: + + No inputs spotted. + Found 1 possible outputs: (name=softmax, op=Softmax) + Found 23885411 (23.89M) const parameters, 0 (0) variable parameters, + and 99 control_edges + Op types used: 489 Const, 99 CheckNumerics, 99 Identity, 94 + BatchNormWithGlobalNormalization, 94 Conv2D, 94 Relu, 11 Concat, 9 AvgPool, + 5 MaxPool, 1 Sub, 1 Softmax, 1 ResizeBilinear, 1 Reshape, 1 Mul, 1 MatMul, + 1 ExpandDims, 1 DecodeJpeg, 1 Cast, 1 BiasAdd + +The important part for our current purposes is the number of const +parameters. In most models these will be stored as 32-bit floats to start, so if +you multiply the number of const parameters by four, you should get something +that’s close to the size of the file on disk. You can often get away with only +eight-bits per parameter with very little loss of accuracy in the final result, +so if your file size is too large you can try using +@{$performance/quantization$quantize_weights} to transform the parameters down. + + bazel build tensorflow/tools/graph_transforms:transform_graph && \ + bazel-bin/tensorflow/tools/graph_transforms/transform_graph \ + --in_graph=/tmp/tensorflow_inception_optimized.pb \ + --out_graph=/tmp/tensorflow_inception_quantized.pb \ + --inputs='Mul:0' --outputs='softmax:0' --transforms='quantize_weights' + +If you look at the resulting file size, you should see that it’s about a quarter +of the original at 23MB. + +Another transform is `round_weights`, which doesn't make the file smaller, but it +makes the file compressable to about the same size as when `quantize_weights` is +used. This is particularly useful for mobile development, taking advantage of +the fact that app bundles are compressed before they’re downloaded by consumers. + +The original file does not compress well with standard algorithms, because the +bit patterns of even very similar numbers can be very different. The +`round_weights` transform keeps the weight parameters stored as floats, but +rounds them to a set number of step values. This means there are a lot more +repeated byte patterns in the stored model, and so compression can often bring +the size down dramatically, in many cases to near the size it would be if they +were stored as eight bit. + +Another advantage of `round_weights` is that the framework doesn’t have to +allocate a temporary buffer to unpack the parameters into, as we have to when +we just use `quantize_weights`. This saves a little bit of latency (though the +results should be cached so it’s only costly on the first run) and makes it +possible to use memory mapping, as described later. + +## Binary Size + +One of the biggest differences between mobile and server development is the +importance of binary size. On desktop machines it’s not unusual to have +executables that are hundreds of megabytes on disk, but for mobile and embedded +apps it’s vital to keep the binary as small as possible so that user downloads +are easy. As mentioned above, TensorFlow only includes a subset of op +implementations by default, but this still results in a 12 MB final +executable. To reduce this, you can set up the library to only include the +implementations of the ops that you actually need, based on automatically +analyzing your model. To use it: + +- Run `tools/print_required_ops/print_selective_registration_header.py` on your + model to produce a header file that only enables the ops it uses. + +- Place the `ops_to_register.h` file somewhere that the compiler can find + it. This can be in the root of your TensorFlow source folder. + +- Build TensorFlow with `SELECTIVE_REGISTRATION` defined, for example by passing + in `--copts=”-DSELECTIVE_REGISTRATION”` to your Bazel build command. + +This process recompiles the library so that only the needed ops and types are +included, which can dramatically reduce the executable size. For example, with +Inception v3, the new size is only 1.5MB. + +## How to Profile your Model + +Once you have an idea of what your device's peak performance range is, it’s +worth looking at its actual current performance. Using a standalone TensorFlow +benchmark, rather than running it inside a larger app, helps isolate just the +Tensorflow contribution to the +latency. The +[tensorflow/tools/benchmark](https://www.tensorflow.org/code/tensorflow/tools/benchmark/) tool +is designed to help you do this. To run it on Inception v3 on your desktop +machine, build this benchmark model: + + bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \ + bazel-bin/tensorflow/tools/benchmark/benchmark_model \ + --graph=/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \ + --input_layer_shape="1,299,299,3" --input_layer_type="float" \ + --output_layer="softmax:0" --show_run_order=false --show_time=false \ + --show_memory=false --show_summary=true --show_flops=true --logtostderr + +You should see output that looks something like this: + +
+============================== Top by Computation Time ==============================
+[node
+ type]  [start]  [first] [avg ms]     [%]  [cdf%]  [mem KB]  [Name]
+Conv2D   22.859   14.212   13.700  4.972%  4.972%  3871.488  conv_4/Conv2D
+Conv2D    8.116    8.964   11.315  4.106%  9.078%  5531.904  conv_2/Conv2D
+Conv2D   62.066   16.504    7.274  2.640% 11.717%   443.904  mixed_3/conv/Conv2D
+Conv2D    2.530    6.226    4.939  1.792% 13.510%  2765.952  conv_1/Conv2D
+Conv2D   55.585    4.605    4.665  1.693% 15.203%   313.600  mixed_2/tower/conv_1/Conv2D
+Conv2D  127.114    5.469    4.630  1.680% 16.883%    81.920  mixed_10/conv/Conv2D
+Conv2D   47.391    6.994    4.588  1.665% 18.548%   313.600  mixed_1/tower/conv_1/Conv2D
+Conv2D   39.463    7.878    4.336  1.574% 20.122%   313.600  mixed/tower/conv_1/Conv2D
+Conv2D  127.113    4.192    3.894  1.413% 21.535%   114.688  mixed_10/tower_1/conv/Conv2D
+Conv2D   70.188    5.205    3.626  1.316% 22.850%   221.952  mixed_4/conv/Conv2D
+
+============================== Summary by node type ==============================
+[Node type]  [count]  [avg ms]    [avg %]    [cdf %]  [mem KB]
+Conv2D            94   244.899    88.952%    88.952% 35869.953
+BiasAdd           95     9.664     3.510%    92.462% 35873.984
+AvgPool            9     7.990     2.902%    95.364%  7493.504
+Relu              94     5.727     2.080%    97.444% 35869.953
+MaxPool            5     3.485     1.266%    98.710%  3358.848
+Const            192     1.727     0.627%    99.337%     0.000
+Concat            11     1.081     0.393%    99.730%  9892.096
+MatMul             1     0.665     0.242%    99.971%     4.032
+Softmax            1     0.040     0.015%    99.986%     4.032
+<>                 1     0.032     0.012%    99.997%     0.000
+Reshape            1     0.007     0.003%   100.000%     0.000
+
+Timings (microseconds): count=50 first=330849 curr=274803 min=232354 max=415352 avg=275563 std=44193
+Memory (bytes): count=50 curr=128366400(all same)
+514 nodes defined 504 nodes observed
+
+ +This is the summary view, which is enabled by the show_summary flag. To +interpret it, the first table is a list of the nodes that took the most time, in +order by how long they took. From left to right, the columns are: + +- Node type, what kind of operation this was. + +- Start time of the op, showing where it falls in the sequence of operations. + +- First time in milliseconds. This is how long the operation took on the first + run of the benchmark, since by default 20 runs are executed to get more + reliable statistics. The first time is useful to spot which ops are doing + expensive calculations on the first run, and then caching the results. + +- Average time for the operation across all runs, in milliseconds. + +- What percentage of the total time for one run the op took. This is useful to + understand where the hotspots are. + +- The cumulative total time of this and the previous ops in the table. This is + handy for understanding what the distribution of work is across the layers, to + see if just a few of the nodes are taking up most of the time. + +- Name of the node. + +The second table is similar, but instead of breaking down the timings by +particular named nodes, it groups them by the kind of op. This is very useful to +understand which op implementations you might want to optimize or eliminate from +your graph. The table is arranged with the most costly operations at the start, +and only shows the top ten entries, with a placeholder for other nodes. The +columns from left to right are: + +- Type of the nodes being analyzed. + +- Accumulated average time taken by all nodes of this type, in milliseconds. + +- What percentage of the total time was taken by this type of operation. + +- Cumulative time taken by this and op types higher in the table, so you can + understand the distribution of the workload. + +- How much memory the outputs of this op type took up. + +Both of these tables are set up so that you can easily copy and paste their +results into spreadsheet documents, since they are output with tabs as +separators between the columns. The summary by node type can be the most useful +when looking for optimization opportunities, since it’s a pointer to the code +that’s taking the most time. In this case, you can see that the Conv2D ops are +almost 90% of the execution time. This is a sign that the graph is pretty +optimal, since convolutions and matrix multiplies are expected to be the bulk of +a neural network’s computing workload. + +As a rule of thumb, it’s more worrying if you see a lot of other operations +taking up more than a small fraction of the time. For neural networks, the ops +that don’t involve large matrix multiplications should usually be dwarfed by the +ones that do, so if you see a lot of time going into those it’s a sign that +either your network is non-optimally constructed, or the code implementing those +ops is not as optimized as it could +be. [Performance bugs](https://github.com/tensorflow/tensorflow/issues) or +patches are always welcome if you do encounter this situation, especially if +they include an attached model exhibiting this behavior and the command line +used to run the benchmark tool on it. + +The run above was on your desktop, but the tool also works on Android, which is +where it’s most useful for mobile development. Here’s an example command line to +run it on a 64-bit ARM device: + + bazel build -c opt --config=android_arm64 \ + tensorflow/tools/benchmark:benchmark_model + adb push bazel-bin/tensorflow/tools/benchmark/benchmark_model /data/local/tmp + adb push /tmp/tensorflow_inception_graph.pb /data/local/tmp/ + adb shell '/data/local/tmp/benchmark_model \ + --graph=/data/local/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \ + --input_layer_shape="1,299,299,3" --input_layer_type="float" \ + --output_layer="softmax:0" --show_run_order=false --show_time=false \ + --show_memory=false --show_summary=true' + +You can interpret the results in exactly the same way as the desktop version +above. If you have any trouble figuring out what the right input and output +names and types are, take a look at the @{$mobile/prepare_models$Preparing +models} page for details about detecting these for your model, and look at the +`summarize_graph` tool which may give you +helpful information. + +There isn’t good support for command line tools on iOS, so instead there’s a +separate example +at +[tensorflow/examples/ios/benchmark](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark) that +packages the same functionality inside a standalone app. This outputs the +statistics to both the screen of the device and the debug log. If you want +on-screen statistics for the Android example apps, you can turn them on by +pressing the volume-up button. + +## Profiling within your own app + +The output you see from the benchmark tool is generated from modules that are +included as part of the standard TensorFlow runtime, which means you have access +to them within your own applications too. You can see an example of how to do +that [here](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark/BenchmarkViewController.mm?l=139). + +The basic steps are: + +1. Create a StatSummarizer object: + + tensorflow::StatSummarizer stat_summarizer(tensorflow_graph); + +2. Set up the options: + + tensorflow::RunOptions run_options; + run_options.set_trace_level(tensorflow::RunOptions::FULL_TRACE); + tensorflow::RunMetadata run_metadata; + +3. Run the graph: + + run_status = session->Run(run_options, inputs, output_layer_names, {}, + output_layers, &run_metadata); + +4. Calculate the results and print them out: + + assert(run_metadata.has_step_stats()); + const tensorflow::StepStats& step_stats = run_metadata.step_stats(); + stat_summarizer->ProcessStepStats(step_stats); + stat_summarizer->PrintStepStats(); + +## Visualizing Models + +The most effective way to speed up your code is by altering your model so it +does less work. To do that, you need to understand what your model is doing, and +visualizing it is a good first step. To get a high-level overview of your graph, +use [TensorBoard](https://github.com/tensorflow/tensorboard). + +## Threading + +The desktop version of TensorFlow has a sophisticated threading model, and will +try to run multiple operations in parallel if it can. In our terminology this is +called “inter-op parallelism” (though to avoid confusion with “intra-op”, you +could think of it as “between-op” instead), and can be set by specifying +`inter_op_parallelism_threads` in the session options. + +By default, mobile devices run operations serially; that is, +`inter_op_parallelism_threads` is set to 1. Mobile processors usually have few +cores and a small cache, so running multiple operations accessing disjoint parts +of memory usually doesn’t help performance. “Intra-op parallelism” (or +“within-op”) can be very helpful though, especially for computation-bound +operations like convolutions where different threads can feed off the same small +set of memory. + +On mobile, how many threads an op will use is set to the number of cores by +default, or 2 when the number of cores can't be determined. You can override the +default number of threads that ops are using by setting +`intra_op_parallelism_threads` in the session options. It’s a good idea to +reduce the default if your app has its own threads doing heavy processing, so +that they don’t interfere with each other. + +To see more details on session options, look at [ConfigProto](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto). + +## Retrain with mobile data + +The biggest cause of accuracy problems when running models on mobile apps is +unrepresentative training data. For example, most of the Imagenet photos are +well-framed so that the object is in the center of the picture, well-lit, and +shot with a normal lens. Photos from mobile devices are often poorly framed, +badly lit, and can have fisheye distortions, especially selfies. + +The solution is to expand your training set with data actually captured from +your application. This step can involve extra work, since you’ll have to label +the examples yourself, but even if you just use it to expand your original +training data, it can help the training set dramatically. Improving the training +set by doing this, and by fixing other quality issues like duplicates or badly +labeled examples is the single best way to improve accuracy. It’s usually a +bigger help than altering your model architecture or using different techniques. + +## Reducing model loading time and/or memory footprint + +Most operating systems allow you to load a file using memory mapping, rather +than going through the usual I/O APIs. Instead of allocating an area of memory +on the heap and then copying bytes from disk into it, you simply tell the +operating system to make the entire contents of a file appear directly in +memory. This has several advantages: + +* Speeds loading +* Reduces paging (increases performance) +* Does not count towards RAM budget for your app + +TensorFlow has support for memory mapping the weights that form the bulk of most +model files. Because of limitations in the `ProtoBuf` serialization format, we +have to make a few changes to our model loading and processing code. The +way memory mapping works is that we have a single file where the first part is a +normal `GraphDef` serialized into the protocol buffer wire format, but then the +weights are appended in a form that can be directly mapped. + +To create this file, run the +`tensorflow/contrib/util:convert_graphdef_memmapped_format` tool. This takes in +a `GraphDef` file that’s been run through `freeze_graph` and converts it to the +format that has the weights appended at the end. Since that file’s no longer a +standard `GraphDef` protobuf, you then need to make some changes to the loading +code. You can see an example of this in +the +[iOS Camera demo app](https://www.tensorflow.org/code/tensorflow/examples/ios/camera/tensorflow_utils.mm?l=147), +in the `LoadMemoryMappedModel()` function. + +The same code (with the Objective C calls for getting the filenames substituted) +can be used on other platforms too. Because we’re using memory mapping, we need +to start by creating a special TensorFlow environment object that’s set up with +the file we’ll be using: + + std::unique_ptr memmapped_env; + memmapped_env->reset( + new tensorflow::MemmappedEnv(tensorflow::Env::Default())); + tensorflow::Status mmap_status = + (memmapped_env->get())->InitializeFromFile(file_path); + +You then need to pass in this environment to subsequent calls, like this one for +loading the graph: + + tensorflow::GraphDef tensorflow_graph; + tensorflow::Status load_graph_status = ReadBinaryProto( + memmapped_env->get(), + tensorflow::MemmappedFileSystem::kMemmappedPackageDefaultGraphDef, + &tensorflow_graph); + +You also need to create the session with a pointer to the environment you’ve +created: + + tensorflow::SessionOptions options; + options.config.mutable_graph_options() + ->mutable_optimizer_options() + ->set_opt_level(::tensorflow::OptimizerOptions::L0); + options.env = memmapped_env->get(); + + tensorflow::Session* session_pointer = nullptr; + tensorflow::Status session_status = + tensorflow::NewSession(options, &session_pointer); + +One thing to notice here is that we’re also disabling automatic optimizations, +since in some cases these will fold constant sub-trees, and so create copies of +tensor values that we don’t want and use up more RAM. + +Once you’ve gone through these steps, you can use the session and graph as +normal, and you should see a reduction in loading time and memory usage. + +## Protecting model files from easy copying + +By default, your models will be stored in the standard serialized protobuf +format on disk. In theory this means that anybody can copy your model, which you +may not want. However, in practice, most models are so application-specific and +obfuscated by optimizations that the risk is similar to that of competitors +disassembling and reusing your code, but if you do want to make it tougher for +casual users to access your files it is possible to take some basic steps. + +Most of our examples use +the +[ReadBinaryProto()](https://www.tensorflow.org/code/tensorflow/core/platform/env.cc?q=core/platform/env.cc&l=409) convenience +call to load a `GraphDef` from disk. This does require an unencrypted protobuf on +disk. Luckily though, the implementation of the call is pretty straightforward +and it should be easy to write an equivalent that can decrypt in memory. Here's +some code that shows how you can read and decrypt a protobuf using your own +decryption routine: + + Status ReadEncryptedProto(Env* env, const string& fname, + ::tensorflow::protobuf::MessageLite* proto) { + string data; + TF_RETURN_IF_ERROR(ReadFileToString(env, fname, &data)); + + DecryptData(&data); // Your own function here. + + if (!proto->ParseFromString(&data)) { + TF_RETURN_IF_ERROR(stream->status()); + return errors::DataLoss("Can't parse ", fname, " as binary proto"); + } + return Status::OK(); + } + +To use this you’d need to define the DecryptData() function yourself. It could +be as simple as something like: + + void DecryptData(string* data) { + for (int i = 0; i < data.size(); ++i) { + data[i] = data[i] ^ 0x23; + } + } + +You may want something more complex, but exactly what you’ll need is outside the +current scope here. diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md new file mode 100644 index 0000000000..c5a560e074 --- /dev/null +++ b/tensorflow/docs_src/mobile/prepare_models.md @@ -0,0 +1,301 @@ +# Preparing models for mobile deployment + +The requirements for storing model information during training are very +different from when you want to release it as part of a mobile app. This section +covers the tools involved in converting from a training model to something +releasable in production. + +## What is up with all the different saved file formats? + +You may find yourself getting very confused by all the different ways that +TensorFlow can save out graphs. To help, here’s a rundown of some of the +different components, and what they are used for. The objects are mostly defined +and serialized as protocol buffers: + +- [NodeDef](https://www.tensorflow.org/code/tensorflow/core/framework/node_def.proto): + Defines a single operation in a model. It has a unique name, a list of the + names of other nodes it pulls inputs from, the operation type it implements + (for example `Add`, or `Mul`), and any attributes that are needed to control + that operation. This is the basic unit of computation for TensorFlow, and all + work is done by iterating through a network of these nodes, applying each one + in turn. One particular operation type that’s worth knowing about is `Const`, + since this holds information about a constant. This may be a single, scalar + number or string, but it can also hold an entire multi-dimensional tensor + array. The values for a `Const` are stored inside the `NodeDef`, and so large + constants can take up a lot of room when serialized. + +- [Checkpoint](https://www.tensorflow.org/code/tensorflow/core/util/tensor_bundle/tensor_bundle.h). Another + way of storing values for a model is by using `Variable` ops. Unlike `Const` + ops, these don’t store their content as part of the `NodeDef`, so they take up + very little space within the `GraphDef` file. Instead their values are held in + RAM while a computation is running, and then saved out to disk as checkpoint + files periodically. This typically happens as a neural network is being + trained and weights are updated, so it’s a time-critical operation, and it may + happen in a distributed fashion across many workers, so the file format has to + be both fast and flexible. They are stored as multiple checkpoint files, + together with metadata files that describe what’s contained within the + checkpoints. When you’re referring to a checkpoint in the API (for example + when passing a filename in as a command line argument), you’ll use the common + prefix for a set of related files. If you had these files: + + /tmp/model/model-chkpt-1000.data-00000-of-00002 + /tmp/model/model-chkpt-1000.data-00001-of-00002 + /tmp/model/model-chkpt-1000.index + /tmp/model/model-chkpt-1000.meta + + You would refer to them as `/tmp/model/chkpt-1000`. + +- [GraphDef](https://www.tensorflow.org/code/tensorflow/core/framework/graph.proto): + Has a list of `NodeDefs`, which together define the computational graph to + execute. During training, some of these nodes will be `Variables`, and so if + you want to have a complete graph you can run, including the weights, you’ll + need to call a restore operation to pull those values from + checkpoints. Because checkpoint loading has to be flexible to deal with all of + the training requirements, this can be tricky to implement on mobile and + embedded devices, especially those with no proper file system available like + iOS. This is where + the + [`freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py) script + comes in handy. As mentioned above, `Const` ops store their values as part of + the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes, + then we only need a single `GraphDef` file to hold the model architecture and + the weights. Freezing the graph handles the process of loading the + checkpoints, and then converts all Consts to Variables. You can then load the + resulting file in a single call, without having to restore variable values + from checkpoints. One thing to watch out for with `GraphDef` files is that + sometimes they’re stored in text format for easy inspection. These versions + usually have a ‘.pbtxt’ filename suffix, whereas the binary files end with + ‘.pb’. + +- [FunctionDefLibrary](https://www.tensorflow.org/code/tensorflow/core/framework/function.proto): + This appears in `GraphDef`, and is effectively a set of sub-graphs, each with + information about their input and output nodes. Each sub-graph can then be + used as an op in the main graph, allowing easy instantiation of different + nodes, in a similar way to how functions encapsulate code in other languages. + +- [MetaGraphDef](https://www.tensorflow.org/code/tensorflow/core/protobuf/meta_graph.proto): + A plain `GraphDef` only has information about the network of computations, but + doesn’t have any extra information about the model or how it can be + used. `MetaGraphDef` contains a `GraphDef` defining the computation part of + the model, but also includes information like ‘signatures’, which are + suggestions about which inputs and outputs you may want to call the model + with, data on how and where any checkpoint files are saved, and convenience + tags for grouping ops together for ease of use. + +- [SavedModel](https://www.tensorflow.org/code/tensorflow/core/protobuf/saved_model.proto): + It’s common to want to have different versions of a graph that rely on a + common set of variable checkpoints. For example, you might need a GPU and a + CPU version of the same graph, but keep the same weights for both. You might + also need some extra files (like label names) as part of your + model. The + [SavedModel](https://www.tensorflow.org/code/tensorflow/python/saved_model/README.md) format + addresses these needs by letting you save multiple versions of the same graph + without duplicating variables, and also storing asset files in the same + bundle. Under the hood, it uses `MetaGraphDef` and checkpoint files, along + with extra metadata files. It’s the format that you’ll want to use if you’re + deploying a web API using TensorFlow Serving, for example. + +## How do you get a model you can use on mobile? + +In most situations, training a model with TensorFlow will give you a folder +containing a `GraphDef` file (usually ending with the `.pb` or `.pbtxt` extension) and +a set of checkpoint files. What you need for mobile or embedded deployment is a +single `GraphDef` file that’s been ‘frozen’, or had its variables converted into +inline constants so everything’s in one file. To handle the conversion, you’ll +need the `freeze_graph.py` script, that’s held in +[`tensorflow/python/tools/freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py). You’ll run it like this: + + bazel build tensorflow/tools:freeze_graph + bazel-bin/tensorflow/tools/freeze_graph \ + --input_graph=/tmp/model/my_graph.pb \ + --input_checkpoint=/tmp/model/model.ckpt-1000 \ + --output_graph=/tmp/frozen_graph.pb \ + --output_node_names=output_node \ + +The `input_graph` argument should point to the `GraphDef` file that holds your +model architecture. It’s possible that your `GraphDef` has been stored in a text +format on disk, in which case it’s likely to end in `.pbtxt` instead of `.pb`, +and you should add an extra `--input_binary=false` flag to the command. + +The `input_checkpoint` should be the most recent saved checkpoint. As mentioned +in the checkpoint section, you need to give the common prefix to the set of +checkpoints here, rather than a full filename. + +`output_graph` defines where the resulting frozen `GraphDef` will be +saved. Because it’s likely to contain a lot of weight values that take up a +large amount of space in text format, it’s always saved as a binary protobuf. + +`output_node_names` is a list of the names of the nodes that you want to extract +the results of your graph from. This is needed because the freezing process +needs to understand which parts of the graph are actually needed, and which are +artifacts of the training process, like summarization ops. Only ops that +contribute to calculating the given output nodes will be kept. If you know how +your graph is going to be used, these should just be the names of the nodes you +pass into `Session::Run()` as your fetch targets. The easiest way to find the +node names is to inspect the Node objects while building your graph in python. +Inspecting your graph in TensorBoard is another simple way. You can get some +suggestions on likely outputs by running the [`summarize_graph` tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms/README.md#inspecting-graphs). + +Because the output format for TensorFlow has changed over time, there are a +variety of other less commonly used flags available too, like `input_saver`, but +hopefully you shouldn’t need these on graphs trained with modern versions of the +framework. + +## Using the Graph Transform Tool + +A lot of the things you need to do to efficiently run a model on device are +available through the [Graph Transform +Tool](https://www.tensorflow.org/code/tensorflow/tools/graph_transforms/README.md). This +command-line tool takes an input `GraphDef` file, applies the set of rewriting +rules you request, and then writes out the result as a `GraphDef`. See the +documentation for more information on how to build and run this tool. + +### Removing training-only nodes + +TensorFlow `GraphDefs` produced by the training code contain all of the +computation that’s needed for back-propagation and updates of weights, as well +as the queuing and decoding of inputs, and the saving out of checkpoints. All of +these nodes are no longer needed during inference, and some of the operations +like checkpoint saving aren’t even supported on mobile platforms. To create a +model file that you can load on devices you need to delete those unneeded +operations by running the `strip_unused_nodes` rule in the Graph Transform Tool. + +The trickiest part of this process is figuring out the names of the nodes you +want to use as inputs and outputs during inference. You'll need these anyway +once you start to run inference, but you also need them here so that the +transform can calculate which nodes are not needed on the inference-only +path. These may not be obvious from the training code. The easiest way to +determine the node name is to explore the graph with TensorBoard. + +Remember that mobile applications typically gather their data from sensors and +have it as arrays in memory, whereas training typically involves loading and +decoding representations of the data stored on disk. In the case of Inception v3 +for example, there’s a `DecodeJpeg` op at the start of the graph that’s designed +to take JPEG-encoded data from a file retrieved from disk and turn it into an +arbitrary-sized image. After that there’s a `BilinearResize` op to scale it to +the expected size, followed by a couple of other ops that convert the byte data +into float and scale the value magnitudes it in the way the rest of the graph +expects. A typical mobile app will skip most of these steps because it’s getting +its input directly from a live camera, so the input node you will actually +supply will be the output of the `Mul` node in this case. + + + +You’ll need to do a similar process of inspection to figure out the correct +output nodes. + +If you’ve just been given a frozen `GraphDef` file, and are not sure about the +contents, try using the `summarize_graph` tool to print out information +about the inputs and outputs it finds from the graph structure. Here’s an +example with the original Inception v3 file: + + bazel run tensorflow/tools/graph_transforms:summarize_graph -- + --in_graph=tensorflow_inception_graph.pb + +Once you have an idea of what the input and output nodes are, you can feed them +into the graph transform tool as the `--input_names` and `--output_names` +arguments, and call the `strip_unused_nodes` transform, like this: + + bazel run tensorflow/tools/graph_transforms:transform_graph -- + --in_graph=tensorflow_inception_graph.pb + --out_graph=optimized_inception_graph.pb --inputs='Mul' --outputs='softmax' + --transforms=' + strip_unused_nodes(type=float, shape="1,299,299,3") + fold_constants(ignore_errors=true) + fold_batch_norms + fold_old_batch_norms' + +One thing to look out for here is that you need to specify the size and type +that you want your inputs to be. This is because any values that you’re going to +be passing in as inputs to inference need to be fed to special `Placeholder` op +nodes, and the transform may need to create them if they don’t already exist. In +the case of Inception v3 for example, a `Placeholder` node replaces the old +`Mul` node that used to output the resized and rescaled image array, since we’re +going to be doing that processing ourselves before we call TensorFlow. It keeps +the original name though, which is why we always feed in inputs to `Mul` when we +run a session with our modified Inception graph. + +After you’ve run this process, you’ll have a graph that only contains the actual +nodes you need to run your prediction process. This is the point where it +becomes useful to run metrics on the graph, so it’s worth running +`summarize_graph` again to understand what’s in your model. + +## What ops should you include on mobile? + +There are hundreds of operations available in TensorFlow, and each one has +multiple implementations for different data types. On mobile platforms, the size +of the executable binary that’s produced after compilation is important, because +app download bundles need to be as small as possible for the best user +experience. If all of the ops and data types are compiled into the TensorFlow +library then the total size of the compiled library can be tens of megabytes, so +by default only a subset of ops and data types are included. + +That means that if you load a model file that’s been trained on a desktop +machine, you may see the error “No OpKernel was registered to support Op” when +you load it on mobile. The first thing to try is to make sure you’ve stripped +out any training-only nodes, since the error will occur at load time even if the +op is never executed. If you’re still hitting the same problem once that’s done, +you’ll need to look at adding the op to your built library. + +The criteria for including ops and types fall into several categories: + +- Are they only useful in back-propagation, for gradients? Since mobile is + focused on inference, we don’t include these. + +- Are they useful mainly for other training needs, such as checkpoint saving? + These we leave out. + +- Do they rely on frameworks that aren’t always available on mobile, such as + libjpeg? To avoid extra dependencies we don’t include ops like `DecodeJpeg`. + +- Are there types that aren’t commonly used? We don’t include boolean variants + of ops for example, since we don’t see much use of them in typical inference + graphs. + +These ops are trimmed by default to optimize for inference on mobile, but it is +possible to alter some build files to change the default. After alternating the +build files, you will need to recompile TensorFlow. See below for more details +on how to do this, and also see @{$mobile/optimizing#binary_size$Optimizing} for +more on reducing your binary size. + +### Locate the implementation + +Operations are broken into two parts. The first is the op definition, which +declares the signature of the operation, which inputs, outputs, and attributes +it has. These take up very little space, and so all are included by default. The +implementations of the op computations are done in kernels, which live in the +`tensorflow/core/kernels` folder. You need to compile the C++ file containing +the kernel implementation of the op you need into the library. To figure out +which file that is, you can search for the operation name in the source +files. + +[Here’s an example search in github](https://github.com/search?utf8=%E2%9C%93&q=repo%3Atensorflow%2Ftensorflow+extension%3Acc+path%3Atensorflow%2Fcore%2Fkernels+REGISTER+Mul&type=Code&ref=searchresults). + +You’ll see that this search is looking for the `Mul` op implementation, and it +finds it in `tensorflow/core/kernels/cwise_op_mul_1.cc`. You need to look for +macros beginning with `REGISTER`, with the op name you care about as one of the +string arguments. + +In this case, the implementations are actually broken up across multiple `.cc` +files, so you’d need to include all of them in your build. If you’re more +comfortable using the command line for code search, here’s a grep command that +also locates the right files if you run it from the root of your TensorFlow +repository: + +`grep 'REGISTER.*"Mul"' tensorflow/core/kernels/*.cc` + +### Add the implementation to the build + +If you’re using Bazel, and building for Android, you’ll want to add the files +you’ve found to +the +[`android_extended_ops_group1`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3565) or +[`android_extended_ops_group2`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3632) targets. You +may also need to include any .cc files they depend on in there. If the build +complains about missing header files, add the .h’s that are needed into +the +[`android_extended_ops`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3525) target. + +If you’re using a makefile targetting iOS, Raspberry Pi, etc, go to +[`tensorflow/contrib/makefile/tf_op_files.txt`](https://www.tensorflow.org/code/tensorflow/contrib/makefile/tf_op_files.txt) and +add the right implementation files there. -- GitLab From bb7ed1c889890ca68f531a3a7b4f56fc55b082df Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 18:44:06 -0700 Subject: [PATCH 550/573] K-FAC: Multi-tower ConvNet example. PiperOrigin-RevId: 173982527 --- tensorflow/contrib/kfac/examples/convnet.py | 194 ++++++++++++------ .../kfac/examples/convnet_mnist_main.py | 12 +- .../kfac/examples/tests/convnet_test.py | 36 ++-- 3 files changed, 158 insertions(+), 84 deletions(-) diff --git a/tensorflow/contrib/kfac/examples/convnet.py b/tensorflow/contrib/kfac/examples/convnet.py index a62780a936..558bc294bc 100644 --- a/tensorflow/contrib/kfac/examples/convnet.py +++ b/tensorflow/contrib/kfac/examples/convnet.py @@ -83,7 +83,7 @@ def conv_layer(layer_id, inputs, kernel_size, out_channels): activations = tf.nn.relu(preactivations) # layer.weights is a list. This converts it a (hashable) tuple. - return preactivations, activations, tuple(layer.weights) + return preactivations, activations, (layer.kernel, layer.bias) def max_pool_layer(layer_id, inputs, kernel_size, stride): @@ -128,7 +128,7 @@ def linear_layer(layer_id, inputs, output_size): return pre, params -def build_model(examples, labels, num_labels, num_ps_tasks=0): +def build_model(examples, labels, num_labels, layer_collection): """Builds a ConvNet classification model. Args: @@ -137,65 +137,64 @@ def build_model(examples, labels, num_labels, num_ps_tasks=0): labels: Tensor of shape [num_examples]. Contains integer IDs to be predicted by softmax for each example. num_labels: int. Number of distinct values 'labels' can take on. - num_ps_tasks: int. Number of parameter servers. If zero, variables - will be placed locally. + layer_collection: LayerCollection instance. Layers will be registered here. Returns: loss: 0-D Tensor representing loss to be minimized. - statistics: dict mapping strings to Tensors. Additional model evaluation - statistics. - layer_collection: LayerCollection instance describing model architecture. + accuracy: 0-D Tensor representing model's accuracy. """ - with tf.device(tf.train.replica_device_setter(num_ps_tasks)): - # Build a ConvNet. For each layer with parameters, we'll keep track of the - # preactivations, activations, weights, and bias. - tf.logging.info("Building model.") - pre0, act0, params0 = conv_layer( - layer_id=0, inputs=examples, kernel_size=5, out_channels=16) - act1 = max_pool_layer(layer_id=1, inputs=act0, kernel_size=3, stride=2) - pre2, act2, params2 = conv_layer( - layer_id=2, inputs=act1, kernel_size=5, out_channels=16) - act3 = max_pool_layer(layer_id=3, inputs=act2, kernel_size=3, stride=2) - flat_act3 = tf.reshape(act3, shape=[-1, int(np.prod(act3.shape[1:4]))]) - logits, params4 = linear_layer( - layer_id=4, inputs=flat_act3, output_size=num_labels) - loss = tf.reduce_mean( - tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits)) - accuracy = tf.reduce_mean( - tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.float32)) - - tf.summary.scalar("loss", loss) - tf.summary.scalar("accuracy", accuracy) - - # Register parameters. K-FAC needs to know about the inputs, outputs, and - # parameters of each conv/fully connected layer and the logits powering the - # posterior probability over classes. - tf.logging.info("Building KFAC Optimizer.") - layer_collection = lc.LayerCollection() - layer_collection.register_conv2d(params0, (1, 1, 1, 1), "SAME", examples, - pre0) - layer_collection.register_conv2d(params2, (1, 1, 1, 1), "SAME", act1, pre2) - layer_collection.register_fully_connected(params4, flat_act3, logits) - layer_collection.register_categorical_predictive_distribution(logits) - - return loss, {"accuracy": accuracy}, layer_collection - - -def minimize_loss_single_machine(loss, statistics, layer_collection): + # Build a ConvNet. For each layer with parameters, we'll keep track of the + # preactivations, activations, weights, and bias. + tf.logging.info("Building model.") + pre0, act0, params0 = conv_layer( + layer_id=0, inputs=examples, kernel_size=5, out_channels=16) + act1 = max_pool_layer(layer_id=1, inputs=act0, kernel_size=3, stride=2) + pre2, act2, params2 = conv_layer( + layer_id=2, inputs=act1, kernel_size=5, out_channels=16) + act3 = max_pool_layer(layer_id=3, inputs=act2, kernel_size=3, stride=2) + flat_act3 = tf.reshape(act3, shape=[-1, int(np.prod(act3.shape[1:4]))]) + logits, params4 = linear_layer( + layer_id=4, inputs=flat_act3, output_size=num_labels) + loss = tf.reduce_mean( + tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=labels, logits=logits)) + accuracy = tf.reduce_mean( + tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.float32)) + + tf.summary.scalar("loss", loss) + tf.summary.scalar("accuracy", accuracy) + + # Register parameters. K-FAC needs to know about the inputs, outputs, and + # parameters of each conv/fully connected layer and the logits powering the + # posterior probability over classes. + tf.logging.info("Building LayerCollection.") + layer_collection.register_conv2d(params0, (1, 1, 1, 1), "SAME", examples, + pre0) + layer_collection.register_conv2d(params2, (1, 1, 1, 1), "SAME", act1, pre2) + layer_collection.register_fully_connected(params4, flat_act3, logits) + layer_collection.register_categorical_predictive_distribution( + logits, name="logits") + + return loss, accuracy + + +def minimize_loss_single_machine(loss, + accuracy, + layer_collection, + session_config=None): """Minimize loss with K-FAC on a single machine. A single Session is responsible for running all of K-FAC's ops. Args: loss: 0-D Tensor. Loss to be minimized. - statistics: dict mapping strings to 0-D Tensors. Additional statistics to - run with each step. + accuracy: 0-D Tensor. Accuracy of classifier on current minibatch. layer_collection: LayerCollection instance describing model architecture. Used by K-FAC to construct preconditioner. + session_config: None or tf.ConfigProto. Configuration for tf.Session(). Returns: - final value for 'statistics'. + final value for 'accuracy'. """ # Train with K-FAC. global_step = tf.train.get_or_create_global_step() @@ -208,19 +207,19 @@ def minimize_loss_single_machine(loss, statistics, layer_collection): train_op = optimizer.minimize(loss, global_step=global_step) tf.logging.info("Starting training.") - with tf.train.MonitoredTrainingSession() as sess: + with tf.train.MonitoredTrainingSession(config=session_config) as sess: while not sess.should_stop(): - global_step_, loss_, statistics_, _, _ = sess.run( - [global_step, loss, statistics, train_op, optimizer.cov_update_op]) + global_step_, loss_, accuracy_, _, _ = sess.run( + [global_step, loss, accuracy, train_op, optimizer.cov_update_op]) if global_step_ % 100 == 0: sess.run(optimizer.inv_update_op) if global_step_ % 100 == 0: - tf.logging.info("global_step: %d | loss: %f | %s", global_step_, loss_, - statistics_) + tf.logging.info("global_step: %d | loss: %f | accuracy: %s", + global_step_, loss_, accuracy_) - return statistics_ + return accuracy_ def _is_gradient_task(task_id, num_tasks): @@ -252,8 +251,7 @@ def _num_gradient_tasks(num_tasks): def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master, - checkpoint_dir, loss, statistics, - layer_collection): + checkpoint_dir, loss, accuracy, layer_collection): """Minimize loss with an synchronous implementation of K-FAC. Different tasks are responsible for different parts of K-FAC's Ops. The first @@ -269,13 +267,13 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master, string to run locally. checkpoint_dir: string or None. Path to store checkpoints under. loss: 0-D Tensor. Loss to be minimized. - statistics: dict mapping strings to 0-D Tensors. Additional statistics to + accuracy: dict mapping strings to 0-D Tensors. Additional accuracy to run with each step. layer_collection: LayerCollection instance describing model architecture. Used by K-FAC to construct preconditioner. Returns: - final value for 'statistics'. + final value for 'accuracy'. Raises: ValueError: if task_id >= num_worker_tasks. @@ -318,12 +316,12 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master, else: raise ValueError("Which op should task %d do?" % task_id) - global_step_, loss_, statistics_, _ = sess.run( - [global_step, loss, statistics, learning_op]) - tf.logging.info("global_step: %d | loss: %f | %s", global_step_, loss_, - statistics_) + global_step_, loss_, accuracy_, _ = sess.run( + [global_step, loss, accuracy, learning_op]) + tf.logging.info("global_step: %d | loss: %f | accuracy: %s", global_step_, + loss_, accuracy_) - return statistics_ + return accuracy_ def train_mnist_single_machine(data_dir, num_epochs, use_fake_data=False): @@ -347,11 +345,69 @@ def train_mnist_single_machine(data_dir, num_epochs, use_fake_data=False): flatten_images=False) # Build a ConvNet. - loss, statistics, layer_collection = build_model( - examples, labels, num_labels=10) + layer_collection = lc.LayerCollection() + loss, accuracy = build_model( + examples, labels, num_labels=10, layer_collection=layer_collection) + + # Fit model. + return minimize_loss_single_machine(loss, accuracy, layer_collection) + + +def train_mnist_multitower(data_dir, num_epochs, num_towers, + use_fake_data=True): + """Train a ConvNet on MNIST. + + Args: + data_dir: string. Directory to read MNIST examples from. + num_epochs: int. Number of passes to make over the training set. + num_towers: int. Number of CPUs to split inference across. + use_fake_data: bool. If True, generate a synthetic dataset. + + Returns: + accuracy of model on the final minibatch of training data. + """ + # Load a dataset. + tf.logging.info("Loading MNIST into memory.") + tower_batch_size = 128 + batch_size = tower_batch_size * num_towers + tf.logging.info( + ("Loading MNIST into memory. Using batch_size = %d = %d towers * %d " + "tower batch size.") % (batch_size, num_towers, tower_batch_size)) + examples, labels = mnist.load_mnist( + data_dir, + num_epochs=num_epochs, + batch_size=batch_size, + use_fake_data=use_fake_data, + flatten_images=False) + + # Split minibatch across towers. + examples = tf.split(examples, num_towers) + labels = tf.split(labels, num_towers) + + # Build an MLP. Each tower's layers will be added to the LayerCollection. + layer_collection = lc.LayerCollection() + tower_results = [] + for tower_id in range(num_towers): + with tf.device("/cpu:%d" % tower_id): + with tf.name_scope("tower%d" % tower_id): + with tf.variable_scope(tf.get_variable_scope(), reuse=(tower_id > 0)): + tf.logging.info("Building tower %d." % tower_id) + tower_results.append( + build_model(examples[tower_id], labels[tower_id], 10, + layer_collection)) + losses, accuracies = zip(*tower_results) + + # Average across towers. + loss = tf.reduce_mean(losses) + accuracy = tf.reduce_mean(accuracies) # Fit model. - return minimize_loss_single_machine(loss, statistics, layer_collection) + session_config = tf.ConfigProto( + allow_soft_placement=False, device_count={ + "CPU": num_towers + }) + return minimize_loss_single_machine( + loss, accuracy, layer_collection, session_config=session_config) def train_mnist_distributed(task_id, @@ -385,13 +441,15 @@ def train_mnist_distributed(task_id, flatten_images=False) # Build a ConvNet. - loss, statistics, layer_collection = build_model( - examples, labels, num_labels=10, num_ps_tasks=num_ps_tasks) + layer_collection = lc.LayerCollection() + with tf.device(tf.train.replica_device_setter(num_ps_tasks)): + loss, accuracy = build_model( + examples, labels, num_labels=10, layer_collection=layer_collection) # Fit model. checkpoint_dir = None if data_dir is None else os.path.join(data_dir, "kfac") return minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, - master, checkpoint_dir, loss, statistics, + master, checkpoint_dir, loss, accuracy, layer_collection) diff --git a/tensorflow/contrib/kfac/examples/convnet_mnist_main.py b/tensorflow/contrib/kfac/examples/convnet_mnist_main.py index 2058c8b6bf..b0c6fbde19 100644 --- a/tensorflow/contrib/kfac/examples/convnet_mnist_main.py +++ b/tensorflow/contrib/kfac/examples/convnet_mnist_main.py @@ -33,7 +33,12 @@ FLAGS = None def main(argv): _ = argv - convnet.train_mnist_single_machine(FLAGS.data_dir, num_epochs=200) + + if FLAGS.num_towers > 1: + convnet.train_mnist_multitower( + FLAGS.data_dir, num_epochs=200, num_towers=FLAGS.num_towers) + else: + convnet.train_mnist_single_machine(FLAGS.data_dir, num_epochs=200) if __name__ == "__main__": @@ -43,5 +48,10 @@ if __name__ == "__main__": type=str, default="/tmp/mnist", help="Directory to store dataset in.") + parser.add_argument( + "--num_towers", + type=int, + default=1, + help="Number of CPUs to split minibatch across.") FLAGS, unparsed = parser.parse_known_args() tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/kfac/examples/tests/convnet_test.py b/tensorflow/contrib/kfac/examples/tests/convnet_test.py index b96dd227e1..3c98c54ef6 100644 --- a/tensorflow/contrib/kfac/examples/tests/convnet_test.py +++ b/tensorflow/contrib/kfac/examples/tests/convnet_test.py @@ -66,8 +66,9 @@ class ConvNetTest(tf.test.TestCase): with tf.Graph().as_default(): x = tf.placeholder(tf.float32, [None, 6, 6, 3]) y = tf.placeholder(tf.int64, [None]) - loss, statistics, layer_collection = convnet.build_model( - x, y, num_labels=5) + layer_collection = lc.LayerCollection() + loss, accuracy = convnet.build_model( + x, y, num_labels=5, layer_collection=layer_collection) # Ensure layers and logits were registered. self.assertEqual(len(layer_collection.fisher_blocks), 3) @@ -80,7 +81,7 @@ class ConvNetTest(tf.test.TestCase): x: np.random.randn(10, 6, 6, 3).astype(np.float32), y: np.random.randint(5, size=10).astype(np.int64), } - sess.run([loss, statistics], feed_dict=feed_dict) + sess.run([loss, accuracy], feed_dict=feed_dict) def _build_toy_problem(self): """Construct a toy linear regression problem. @@ -90,8 +91,7 @@ class ConvNetTest(tf.test.TestCase): Returns: loss: 0-D Tensor representing loss to be minimized. - statistics: dict mapping strings to Tensors. Additional model evaluation - statistics. + accuracy: 0-D Tensors representing model accuracy. layer_collection: LayerCollection instance describing model architecture. """ x = np.asarray([[1.], [2.]]).astype(np.float32) @@ -101,34 +101,34 @@ class ConvNetTest(tf.test.TestCase): w = tf.get_variable("w", shape=[1, 1], initializer=tf.zeros_initializer()) y_hat = tf.matmul(x, w) loss = tf.reduce_mean(0.5 * tf.square(y_hat - y)) - statistics = {"loss": loss} + accuracy = loss layer_collection = lc.LayerCollection() layer_collection.register_fully_connected(params=w, inputs=x, outputs=y_hat) layer_collection.register_normal_predictive_distribution(y_hat) - return loss, statistics, layer_collection + return loss, accuracy, layer_collection def testMinimizeLossSingleMachine(self): with tf.Graph().as_default(): - loss, statistics, layer_collection = self._build_toy_problem() - statistics_ = convnet.minimize_loss_single_machine( - loss, statistics, layer_collection) - self.assertLess(statistics_["loss"], 1.0) + loss, accuracy, layer_collection = self._build_toy_problem() + accuracy_ = convnet.minimize_loss_single_machine(loss, accuracy, + layer_collection) + self.assertLess(accuracy_, 1.0) def testMinimizeLossDistributed(self): with tf.Graph().as_default(): - loss, statistics, layer_collection = self._build_toy_problem() - statistics_ = convnet.minimize_loss_distributed( + loss, accuracy, layer_collection = self._build_toy_problem() + accuracy_ = convnet.minimize_loss_distributed( task_id=0, num_worker_tasks=1, num_ps_tasks=0, master="", checkpoint_dir=None, loss=loss, - statistics=statistics, + accuracy=accuracy, layer_collection=layer_collection) - self.assertLess(statistics_["loss"], 1.0) + self.assertLess(accuracy_, 1.0) def testTrainMnistSingleMachine(self): with tf.Graph().as_default(): @@ -140,6 +140,12 @@ class ConvNetTest(tf.test.TestCase): convnet.train_mnist_single_machine( data_dir=None, num_epochs=1, use_fake_data=True) + def testTrainMnistMultitower(self): + with tf.Graph().as_default(): + # Ensure model training doesn't crash. + convnet.train_mnist_multitower( + data_dir=None, num_epochs=1, num_towers=2, use_fake_data=True) + def testTrainMnistDistributed(self): with tf.Graph().as_default(): # Ensure model training doesn't crash. -- GitLab From d9cee35b66440a00d2582d6043a6f6d4007bae6e Mon Sep 17 00:00:00 2001 From: LevineHuang Date: Tue, 31 Oct 2017 09:54:14 +0800 Subject: [PATCH 551/573] Typo fix in file 'fully_connected_feed.py' (#14033) * Typo fix in file 'fully_connected_feed.py' * Minor edits to coding style --- tensorflow/examples/tutorials/mnist/fully_connected_feed.py | 2 +- tensorflow/examples/tutorials/mnist/mnist_deep.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py index af89c8c77b..35ca1b2f7f 100644 --- a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py +++ b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py @@ -109,7 +109,7 @@ def do_eval(sess, labels_placeholder) true_count += sess.run(eval_correct, feed_dict=feed_dict) precision = float(true_count) / num_examples - print(' Num examples: %d Num correct: %d Precision @ 1: %0.04f' % + print('Num examples: %d Num correct: %d Precision @ 1: %0.04f' % (num_examples, true_count, precision)) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 4b5b50400a..a4dbab5123 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -82,7 +82,7 @@ def deepnn(x): W_fc1 = weight_variable([7 * 7 * 64, 1024]) b_fc1 = bias_variable([1024]) - h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64]) + h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) # Dropout - controls the complexity of the model, prevents co-adaptation of -- GitLab From a4b5356e476016e0f537766ac2ac891eab9900e1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 18:58:48 -0700 Subject: [PATCH 552/573] [TF:XLA] Reduce boilerplate code in HLO visitors. Only pass the HloInstruction into visitor methods. This makes changing instructions and visitors easier. PiperOrigin-RevId: 173983398 --- .../xla/service/algebraic_simplifier.cc | 167 ++++++++--------- .../compiler/xla/service/cpu/cpu_compiler.cc | 9 +- .../compiler/xla/service/cpu/ir_emitter.cc | 90 +++++---- .../compiler/xla/service/cpu/ir_emitter.h | 58 ++---- .../compiler/xla/service/dfs_hlo_visitor.h | 139 +++++--------- .../service/dfs_hlo_visitor_with_default.h | 81 ++------ .../compiler/xla/service/gpu/ir_emitter.cc | 60 +++--- .../compiler/xla/service/gpu/ir_emitter.h | 59 ++---- .../xla/service/gpu/ir_emitter_unnested.cc | 48 ++--- .../compiler/xla/service/hlo_cost_analysis.cc | 141 +++++--------- .../compiler/xla/service/hlo_cost_analysis.h | 69 ++----- .../compiler/xla/service/hlo_evaluator.cc | 177 ++++++++---------- .../compiler/xla/service/hlo_evaluator.h | 20 +- .../compiler/xla/service/hlo_instruction.cc | 97 +++++----- .../xla/service/hlo_instruction_test.cc | 28 ++- .../compiler/xla/service/hlo_verifier.cc | 96 ++++------ tensorflow/compiler/xla/service/inliner.cc | 14 +- .../xla/service/llvm_ir/fused_ir_emitter.cc | 14 +- .../xla/service/llvm_ir/fused_ir_emitter.h | 10 +- .../xla/service/logical_buffer_analysis.cc | 14 +- .../xla/service/logical_buffer_analysis.h | 11 +- .../xla/service/tuple_points_to_analysis.cc | 17 +- .../xla/service/tuple_points_to_analysis.h | 11 +- 23 files changed, 537 insertions(+), 893 deletions(-) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 2a610e91f0..ee5cf8a100 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -123,74 +123,54 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleAdd(HloInstruction* add) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleBroadcast(HloInstruction* broadcast) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; Status HandleCopy(HloInstruction* copy) override; Status HandleConvert(HloInstruction* convert) override; - Status HandleReal(HloInstruction* real, HloInstruction* operand) override; - Status HandleImag(HloInstruction* imag, HloInstruction* operand) override; + Status HandleReal(HloInstruction* real) override; + Status HandleImag(HloInstruction* imag) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleConvolution(HloInstruction* convolution) override; - Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleDivide(HloInstruction* divide) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleDot(HloInstruction* dot) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; - Status HandleLog(HloInstruction* log, HloInstruction* operand) override; + Status HandleLog(HloInstruction* log) override; - Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleMultiply(HloInstruction* multiply) override; Status HandlePad(HloInstruction* pad) override; - Status HandlePower(HloInstruction* power, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandlePower(HloInstruction* power) override; Status HandleReshape(HloInstruction* reshape) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; - - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override; - - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override; - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override; - Status HandleDynamicSlice(HloInstruction* slice, HloInstruction* operand, - HloInstruction* start_indices) override; - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override; + Status HandleReduce(HloInstruction* reduce) override; + + Status HandleReduceWindow(HloInstruction* reduce_window) override; + + Status HandleReverse(HloInstruction* reverse) override; + Status HandleSlice(HloInstruction* slice) override; + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override; + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override; Status HandleTranspose(HloInstruction* transpose) override; - Status HandleSubtract(HloInstruction* sub, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleSubtract(HloInstruction* sub) override; Status HandleMaximum(HloInstruction* maximum) override; Status HandleMinimum(HloInstruction* minimum) override; @@ -339,9 +319,9 @@ bool AlgebraicSimplifierVisitor::ReplaceInstructionIfSameShape( return true; } -Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add) { + auto lhs = add->mutable_operand(0); + auto rhs = add->mutable_operand(1); // A + 0 => A VLOG(10) << "trying transform [A + 0 => A]: " << add->ToString(); if (IsAll(rhs, 0) && ReplaceInstructionIfSameShape(add, lhs)) { @@ -384,8 +364,9 @@ Status AlgebraicSimplifierVisitor::HandleCopy(HloInstruction* copy) { } Status AlgebraicSimplifierVisitor::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { + HloInstruction* concatenate) { + tensorflow::gtl::ArraySlice operands( + concatenate->operands()); if (operands.size() == 1) { // Unary concatenates are useless. ReplaceInstructionIfSameShape(concatenate, operands[0]); @@ -466,20 +447,19 @@ static HloInstruction* BuildTupleConstant(HloComputation* computation, } } -Status AlgebraicSimplifierVisitor::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status AlgebraicSimplifierVisitor::HandleConstant(HloInstruction* constant) { // Tuple constants aren't directly supported by any backend. Expand them into // explicit Tuple instructions. if (ShapeUtil::IsTuple(constant->shape())) { - return ReplaceInstruction(constant, - BuildTupleConstant(computation_, literal)); + return ReplaceInstruction( + constant, BuildTupleConstant(computation_, constant->literal())); } return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleSubtract(HloInstruction* sub, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleSubtract(HloInstruction* sub) { + auto lhs = sub->mutable_operand(0); + auto rhs = sub->mutable_operand(1); // A - 0 => A VLOG(10) << "trying transform [A - 0 => A]: " << sub->ToString(); if (IsAll(rhs, 0) && ReplaceInstructionIfSameShape(sub, lhs)) { @@ -489,9 +469,9 @@ Status AlgebraicSimplifierVisitor::HandleSubtract(HloInstruction* sub, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) { + auto lhs = divide->mutable_operand(0); + auto rhs = divide->mutable_operand(1); // A/1 => A VLOG(10) << "trying transform [A/1 => A]: " << divide->ToString(); if (IsAll(rhs, 1) && ReplaceInstructionIfSameShape(divide, lhs)) { @@ -598,9 +578,9 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) { + auto lhs = dot->mutable_operand(0); + auto rhs = dot->mutable_operand(1); if (!enable_dot_simplification_) { return Status::OK(); } @@ -729,9 +709,9 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply) { + auto lhs = multiply->mutable_operand(0); + auto rhs = multiply->mutable_operand(1); // A*1 => A VLOG(10) << "trying transform [A*1 => A]: " << multiply->ToString(); if (IsAll(rhs, 1) && ReplaceInstructionIfSameShape(multiply, lhs)) { @@ -755,10 +735,10 @@ Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleLog(HloInstruction* log, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleLog(HloInstruction* log) { // ln(exp(A)) => A VLOG(10) << "trying transform [ln(exp(A)) => A]: " << log->ToString(); + auto operand = log->mutable_operand(0); if (operand->opcode() == HloOpcode::kExp && ReplaceInstructionIfSameShape(log, operand->mutable_operand(0))) { return Status::OK(); @@ -778,7 +758,8 @@ Status AlgebraicSimplifierVisitor::HandleLog(HloInstruction* log, } Status AlgebraicSimplifierVisitor::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { + HloInstruction* get_tuple_element) { + auto operand = get_tuple_element->mutable_operand(0); if (operand->opcode() == HloOpcode::kTuple) { // get_tuple_element(make_tuple({A_0, A_1, ..., A_n}), i) => A_i VLOG(10) << "trying transform " @@ -971,8 +952,8 @@ Status AlgebraicSimplifierVisitor::HandleConvert(HloInstruction* convert) { } // Real(Complex(r, i)) -> r -Status AlgebraicSimplifierVisitor::HandleReal(HloInstruction* real, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleReal(HloInstruction* real) { + auto operand = real->mutable_operand(0); if (operand->opcode() == HloOpcode::kComplex) { return ReplaceInstruction(real, operand->mutable_operand(0)); } @@ -980,8 +961,8 @@ Status AlgebraicSimplifierVisitor::HandleReal(HloInstruction* real, } // Imag(Complex(r, i)) -> i -Status AlgebraicSimplifierVisitor::HandleImag(HloInstruction* imag, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleImag(HloInstruction* imag) { + auto operand = imag->mutable_operand(0); if (operand->opcode() == HloOpcode::kComplex) { return ReplaceInstruction(imag, operand->mutable_operand(1)); } @@ -1078,10 +1059,10 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) { return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandlePower(HloInstruction* power, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandlePower(HloInstruction* power) { VLOG(10) << "trying transform [pow(A, 0) => 1]: " << power->ToString(); + auto lhs = power->mutable_operand(0); + auto rhs = power->mutable_operand(1); if (IsAll(rhs, 0)) { auto one = HloInstruction::CreateConstant( Literal::One(power->shape().element_type()).CloneToUnique()); @@ -1265,8 +1246,7 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) { return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse) { // When all the dimensions to reverse are trivial (i.e. the bound is 1), // there is nothing to be done. auto dim_is_one = [&](int64 i) -> bool { @@ -1274,23 +1254,23 @@ Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse, }; if (std::all_of(reverse->dimensions().begin(), reverse->dimensions().end(), dim_is_one)) { - return ReplaceInstruction(reverse, operand); + return ReplaceInstruction(reverse, reverse->mutable_operand(0)); } return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleSlice(HloInstruction* slice, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleSlice(HloInstruction* slice) { // Delete no-op slices, i.e. where shape = operand shape. - if (ReplaceInstructionIfSameShape(slice, operand)) { + if (ReplaceInstructionIfSameShape(slice, slice->mutable_operand(0))) { return Status::OK(); } return Status::OK(); } Status AlgebraicSimplifierVisitor::HandleDynamicSlice( - HloInstruction* dynamic_slice, HloInstruction* operand, - HloInstruction* start_indices) { + HloInstruction* dynamic_slice) { + auto operand = dynamic_slice->mutable_operand(0); + auto start_indices = dynamic_slice->operand(1); if (ShapeUtil::IsScalar(dynamic_slice->shape())) { return ReplaceInstruction(dynamic_slice, operand); } @@ -1303,8 +1283,9 @@ Status AlgebraicSimplifierVisitor::HandleDynamicSlice( } Status AlgebraicSimplifierVisitor::HandleDynamicUpdateSlice( - HloInstruction* dynamic_update_slice, HloInstruction* operand, - HloInstruction* update, HloInstruction* start_indices) { + HloInstruction* dynamic_update_slice) { + auto update = dynamic_update_slice->mutable_operand(1); + auto start_indices = dynamic_update_slice->operand(2); // DynamicUpdateSlice on a scalar just passes through the update argument. if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) { return ReplaceInstruction(dynamic_update_slice, update); @@ -1323,9 +1304,11 @@ Status AlgebraicSimplifierVisitor::HandleDynamicUpdateSlice( return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleReduce( - HloInstruction* reduce, HloInstruction* arg, HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, HloComputation* function) { +Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->mutable_operand(0); + auto init_value = reduce->mutable_operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); if (ShapeUtil::HasZeroElements(arg->shape()) || ShapeUtil::HasZeroElements(reduce->shape())) { return ReplaceWithNewInstruction( @@ -1403,8 +1386,10 @@ Status AlgebraicSimplifierVisitor::HandleReduce( } Status AlgebraicSimplifierVisitor::HandleReduceWindow( - HloInstruction* reduce_window, HloInstruction* operand, - const Window& window, HloComputation* function) { + HloInstruction* reduce_window) { + auto operand = reduce_window->mutable_operand(0); + const Window& window = reduce_window->window(); + auto function = reduce_window->to_apply(); VLOG(10) << "Considering folding Pad: " << operand->ToString() << "\ninto reduce-window: " << reduce_window->ToString(); @@ -1487,8 +1472,10 @@ Status AlgebraicSimplifierVisitor::HandleTranspose(HloInstruction* transpose) { } Status AlgebraicSimplifierVisitor::HandleConvolution( - HloInstruction* convolution, HloInstruction* lhs, HloInstruction* rhs, - const Window& window) { + HloInstruction* convolution) { + auto lhs = convolution->mutable_operand(0); + auto rhs = convolution->mutable_operand(1); + const auto& window = convolution->window(); if (!enable_conv_simplification_) { return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 65e117e68f..e141066b8f 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -222,14 +222,9 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault { } // Skip constants, there is nothing to profile. - Status HandleConstant(HloInstruction* /*constant*/, - const Literal& /*literal*/) override { - return Status::OK(); - } + Status HandleConstant(HloInstruction*) override { return Status::OK(); } // Skip parameters, they are a simple load. - Status HandleParameter(HloInstruction* /*parameter*/) override { - return Status::OK(); - } + Status HandleParameter(HloInstruction*) override { return Status::OK(); } // It is important to recurse for "while" or else we risk overly coarse // profiling information. Status HandleWhile(HloInstruction* xla_while) override { diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index fa3b3ab8e7..a20ce6826c 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -262,9 +262,9 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } -Status IrEmitter::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status IrEmitter::HandleConstant(HloInstruction* constant) { VLOG(2) << "HandleConstant: " << constant->ToString(); + const Literal& literal = constant->literal(); llvm::GlobalVariable* global_for_const; // We avoid creating large constants in the LLVM IR since LLVM is not @@ -392,12 +392,12 @@ void IrEmitter::AttachDereferenceableMetadataForLoad(llvm::LoadInst* load, } } -Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) { // A tuple is an array of pointers, one for each operand. Each pointer points // to the output buffer of its corresponding operand. A GetTupleElement // instruction forwards a pointer to the tuple element buffer at the given // index. + auto operand = get_tuple_element->operand(0); const Shape& shape = get_tuple_element->shape(); emitted_value_[get_tuple_element] = llvm_ir::EmitGetTupleElement( shape, get_tuple_element->tuple_index(), MinimumAlignmentForShape(shape), @@ -405,9 +405,10 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, return Status::OK(); } -Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { +Status IrEmitter::HandleSelect(HloInstruction* select) { + auto pred = select->operand(0); + auto on_true = select->operand(1); + auto on_false = select->operand(2); TF_RET_CHECK(pred->shape().element_type() == PRED); if (ShapeUtil::IsTuple(select->shape())) { @@ -571,27 +572,24 @@ Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) { return Status::OK(); } -Status IrEmitter::HandleSort(HloInstruction* sort, HloInstruction* operand) { +Status IrEmitter::HandleSort(HloInstruction* sort) { // TODO(b/26783907): Implement sort on CPU. return Unimplemented("Sort is not supported on CPU (b/26783907)."); } -Status IrEmitter::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitter::HandleTuple(HloInstruction* tuple) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(tuple)); std::vector base_ptrs; - for (auto operand : operands) { + for (auto operand : tuple->operands()) { base_ptrs.push_back(GetEmittedValueFor(operand)); } llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_, module_); return Status::OK(); } -Status IrEmitter::HandleMap( - HloInstruction* map, tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice /*static_operands*/) { +Status IrEmitter::HandleMap(HloInstruction* map) { + tensorflow::gtl::ArraySlice operands(map->operands()); + HloComputation* function = map->to_apply(); // The called computation should have been emitted previously. llvm::Function* mapped_ir_function = FindOrDie(emitted_functions_, function); @@ -608,10 +606,10 @@ Status IrEmitter::HandleMap( }); } -Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, - const Window& window, - HloComputation* function) { +Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window) { + auto operand = reduce_window->operand(0); + const Window& window = reduce_window->window(); + HloComputation* function = reduce_window->to_apply(); TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*reduce_window, /*operands=*/{operand}, /*supported_types=*/{F32})); @@ -892,8 +890,9 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) { return Status::OK(); } -Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) { +Status IrEmitter::HandleDot(HloInstruction* dot) { + auto lhs = dot->operand(0); + auto rhs = dot->operand(1); TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*dot, /*operands=*/{lhs, rhs}, /*supported_types=*/{F32, F64, C64})); @@ -919,9 +918,10 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs, hlo_module_config_); } -Status IrEmitter::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs, HloInstruction* rhs, - const Window& window) { +Status IrEmitter::HandleConvolution(HloInstruction* convolution) { + auto lhs = convolution->operand(0); + auto rhs = convolution->operand(1); + const auto& window = convolution->window(); TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*convolution, /*operands=*/{lhs, rhs}, /*supported_types=*/{F32, C64})); @@ -1900,10 +1900,11 @@ StatusOr IrEmitter::EmitVectorizedReduce( return true; } -Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) { +Status IrEmitter::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->mutable_operand(0); + auto init_value = reduce->mutable_operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); if (!options::VectorizedReduceDisabled(hlo_module_config_)) { string vectorization_failure_reason; TF_ASSIGN_OR_RETURN( @@ -1982,9 +1983,9 @@ Status IrEmitter::HandleSend(HloInstruction* send) { return Unimplemented("Send is not implemented on CPU. See b/33942983."); } -Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) { +Status IrEmitter::HandleSlice(HloInstruction* slice) { VLOG(2) << "HandleSlice: " << slice->ToString(); - + auto operand = slice->operand(0); // The code below emits a sequential loop nest. For the parallel backend, use // EmitParallelTargetElementLoop() which respects dynamic loop bounds. if (ShouldEmitParallelLoopFor(*slice)) { @@ -2117,20 +2118,17 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) { return Status::OK(); } -Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* /*start_indices*/) { +Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice) { if (ShapeUtil::IsScalar(dynamic_slice->shape())) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_slice)); - return EmitMemcpy(*operand, *dynamic_slice); + return EmitMemcpy(*dynamic_slice->operand(0), *dynamic_slice); } return DefaultAction(dynamic_slice); } -Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) { +Status IrEmitter::HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) { + auto update = dynamic_update_slice->operand(1); if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice)); return EmitMemcpy(*update, *dynamic_update_slice); @@ -2305,10 +2303,10 @@ Status IrEmitter::HandleCall(HloInstruction* call) { return Status::OK(); } -Status IrEmitter::HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) { +Status IrEmitter::HandleCustomCall(HloInstruction* custom_call) { + tensorflow::gtl::ArraySlice operands( + custom_call->operands()); + tensorflow::StringPiece custom_call_target(custom_call->custom_call_target()); llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy(); llvm::AllocaInst* operands_alloca = llvm_ir::EmitAllocaAtFunctionEntryWithCount( @@ -2578,9 +2576,9 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source, } } -Status IrEmitter::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitter::HandleConcatenate(HloInstruction* concatenate) { + tensorflow::gtl::ArraySlice operands( + concatenate->operands()); string failure_reason; TF_ASSIGN_OR_RETURN( bool successful, diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 58c185af1e..5d061e11e3 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -154,62 +154,36 @@ class IrEmitter : public DfsHloVisitorWithDefault { Status DefaultAction(HloInstruction* hlo) override; Status HandleBitcast(HloInstruction* bitcast) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleSelect(HloInstruction* select) override; + Status HandleDot(HloInstruction* dot) override; + Status HandleConvolution(HloInstruction* convolution) override; Status HandleBatchNormTraining(HloInstruction* batch_norm_training) override; Status HandleBatchNormGrad(HloInstruction* batch_norm_grad) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; Status HandleInfeed(HloInstruction* infeed) override; Status HandleOutfeed(HloInstruction* outfeed) override; - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override; + Status HandleSort(HloInstruction* sort) override; Status HandleParameter(HloInstruction* parameter) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override; + Status HandleReduce(HloInstruction* reduce) override; + Status HandleReduceWindow(HloInstruction* reduce_window) override; Status HandleSelectAndScatter(HloInstruction* select_and_scatter) override; Status HandleSend(HloInstruction* send) override; - Status HandleSlice(HloInstruction* slice, - HloInstruction* /*operand*/) override; - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* /*operand*/, - HloInstruction* /*start_indices*/) override; - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* /*operand*/, - HloInstruction* /*update*/, - HloInstruction* /*start_indices*/) override; + Status HandleSlice(HloInstruction* slice) override; + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override; + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override; Status HandleRecv(HloInstruction* recv) override; Status HandlePad(HloInstruction* pad) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleMap(HloInstruction* map) override; Status HandleFusion(HloInstruction* fusion) override; Status HandleCall(HloInstruction* call) override; - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override; + Status HandleCustomCall(HloInstruction* custom_call) override; Status HandleWhile(HloInstruction* xla_while) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; Status FinishVisit(HloInstruction* root) override; Status Preprocess(HloInstruction* hlo) override; diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h index adaff90913..e57a492dde 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h @@ -65,65 +65,49 @@ class DfsHloVisitor { virtual Status HandleElementwiseUnary(HloInstruction* hlo); virtual Status HandleElementwiseBinary(HloInstruction* hlo); - virtual Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) = 0; - virtual Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) = 0; + virtual Status HandleClamp(HloInstruction* clamp) = 0; + virtual Status HandleSelect(HloInstruction* select) = 0; virtual Status HandleMaximum(HloInstruction* maximum) { return HandleElementwiseBinary(maximum); } virtual Status HandleMinimum(HloInstruction* minimum) { return HandleElementwiseBinary(minimum); } - virtual Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) = 0; + virtual Status HandleConcatenate(HloInstruction* concatenate) = 0; virtual Status HandleConvert(HloInstruction* convert) { return HandleElementwiseUnary(convert); } virtual Status HandleCopy(HloInstruction* copy) { return HandleElementwiseUnary(copy); } - virtual Status HandleComplex(HloInstruction* complex, HloInstruction* real, - HloInstruction* imag) { + virtual Status HandleComplex(HloInstruction* complex) { return HandleElementwiseBinary(complex); } - virtual Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleMultiply(HloInstruction* multiply) { return HandleElementwiseBinary(multiply); } - virtual Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) = 0; - virtual Status HandlePower(HloInstruction* power, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleDot(HloInstruction* dot) = 0; + virtual Status HandlePower(HloInstruction* power) { return HandleElementwiseBinary(power); } - virtual Status HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs, HloInstruction* rhs, - const Window& window) = 0; + virtual Status HandleConvolution(HloInstruction* convolution) = 0; virtual Status HandleCrossReplicaSum(HloInstruction* crs) = 0; - virtual Status HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) { + virtual Status HandleCompare(HloInstruction* compare) { return HandleElementwiseBinary(compare); } - virtual Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleAdd(HloInstruction* add) { return HandleElementwiseBinary(add); } - virtual Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleDivide(HloInstruction* divide) { return HandleElementwiseBinary(divide); } - virtual Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleRemainder(HloInstruction* remainder) { return HandleElementwiseBinary(remainder); } - virtual Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleSubtract(HloInstruction* subtract) { return HandleElementwiseBinary(subtract); } - virtual Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { + virtual Status HandleAbs(HloInstruction* abs) { return HandleElementwiseUnary(abs); } virtual Status HandleAtan2(HloInstruction* atan2, HloInstruction* y, @@ -133,66 +117,59 @@ class DfsHloVisitor { virtual Status HandleRound(HloInstruction* round) { return HandleElementwiseUnary(round); } - virtual Status HandleSign(HloInstruction* sign, HloInstruction* operand) { + virtual Status HandleSign(HloInstruction* sign) { return HandleElementwiseUnary(sign); } - virtual Status HandleNegate(HloInstruction* negate, HloInstruction* operand) { + virtual Status HandleNegate(HloInstruction* negate) { return HandleElementwiseUnary(negate); } - virtual Status HandleExp(HloInstruction* exp, HloInstruction* operand) { + virtual Status HandleExp(HloInstruction* exp) { return HandleElementwiseUnary(exp); } - virtual Status HandleFloor(HloInstruction* floor, HloInstruction* operand) { + virtual Status HandleFloor(HloInstruction* floor) { return HandleElementwiseUnary(floor); } - virtual Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) { + virtual Status HandleCeil(HloInstruction* ceil) { return HandleElementwiseUnary(ceil); } - virtual Status HandleLog(HloInstruction* log, HloInstruction* operand) { + virtual Status HandleLog(HloInstruction* log) { return HandleElementwiseUnary(log); } - virtual Status HandleCos(HloInstruction* cos, HloInstruction* operand) { + virtual Status HandleCos(HloInstruction* cos) { return HandleElementwiseUnary(cos); } - virtual Status HandleSin(HloInstruction* sin, HloInstruction* operand) { + virtual Status HandleSin(HloInstruction* sin) { return HandleElementwiseUnary(sin); } - virtual Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) { + virtual Status HandleTanh(HloInstruction* tanh) { return HandleElementwiseUnary(tanh); } - virtual Status HandleReal(HloInstruction* real, HloInstruction* operand) { + virtual Status HandleReal(HloInstruction* real) { return HandleElementwiseUnary(real); } - virtual Status HandleImag(HloInstruction* imag, HloInstruction* operand) { + virtual Status HandleImag(HloInstruction* imag) { return HandleElementwiseUnary(imag); } - virtual Status HandleIsFinite(HloInstruction* is_finite, - HloInstruction* operand) { + virtual Status HandleIsFinite(HloInstruction* is_finite) { return HandleElementwiseUnary(is_finite); } - virtual Status HandleAnd(HloInstruction* and_, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleAnd(HloInstruction* and_) { return HandleElementwiseBinary(and_); } - virtual Status HandleNot(HloInstruction* not_, HloInstruction* operand) { + virtual Status HandleNot(HloInstruction* not_) { return HandleElementwiseUnary(not_); } - virtual Status HandleOr(HloInstruction* or_, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleOr(HloInstruction* or_) { return HandleElementwiseBinary(or_); } - virtual Status HandleShiftLeft(HloInstruction* shift_left, - HloInstruction* lhs, HloInstruction* rhs) { + virtual Status HandleShiftLeft(HloInstruction* shift_left) { return HandleElementwiseBinary(shift_left); } virtual Status HandleShiftRightArithmetic( - HloInstruction* shift_right_arithmetic, HloInstruction* lhs, - HloInstruction* rhs) { + HloInstruction* shift_right_arithmetic) { return HandleElementwiseBinary(shift_right_arithmetic); } - virtual Status HandleShiftRightLogical(HloInstruction* shift_right_logical, - HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleShiftRightLogical(HloInstruction* shift_right_logical) { return HandleElementwiseBinary(shift_right_logical); } @@ -202,19 +179,12 @@ class DfsHloVisitor { virtual Status HandleInfeed(HloInstruction* infeed) = 0; virtual Status HandleOutfeed(HloInstruction* outfeed) = 0; - virtual Status HandleRng(HloInstruction* random, - RandomDistribution distribution) = 0; - virtual Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) = 0; - virtual Status HandleSort(HloInstruction* sort, HloInstruction* operand) = 0; - virtual Status HandleConstant(HloInstruction* constant, - const Literal& literal) = 0; - virtual Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) = 0; - virtual Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) = 0; + virtual Status HandleRng(HloInstruction* random) = 0; + virtual Status HandleReverse(HloInstruction* reverse) = 0; + virtual Status HandleSort(HloInstruction* sort) = 0; + virtual Status HandleConstant(HloInstruction* constant) = 0; + virtual Status HandleGetTupleElement(HloInstruction* get_tuple_element) = 0; + virtual Status HandleReduce(HloInstruction* reduce) = 0; virtual Status HandleBitcast(HloInstruction* bitcast) = 0; virtual Status HandleBroadcast(HloInstruction* broadcast) = 0; virtual Status HandleReshape(HloInstruction* reshape) = 0; @@ -222,31 +192,14 @@ class DfsHloVisitor { virtual Status HandleParameter(HloInstruction* parameter) = 0; virtual Status HandleFusion(HloInstruction* fusion) = 0; virtual Status HandleCall(HloInstruction* call) = 0; - virtual Status HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) = 0; - virtual Status HandleSlice(HloInstruction* slice, - HloInstruction* operand) = 0; - virtual Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) = 0; - virtual Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) = 0; - virtual Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) = 0; - virtual Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) = 0; - virtual Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, - const Window& window, - HloComputation* function) = 0; + virtual Status HandleCustomCall(HloInstruction* custom_call) = 0; + virtual Status HandleSlice(HloInstruction* slice) = 0; + virtual Status HandleDynamicSlice(HloInstruction* dynamic_slice) = 0; + virtual Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) = 0; + virtual Status HandleTuple(HloInstruction* tuple) = 0; + virtual Status HandleMap(HloInstruction* map) = 0; + virtual Status HandleReduceWindow(HloInstruction* reduce_window) = 0; virtual Status HandleSelectAndScatter(HloInstruction* instruction) = 0; virtual Status HandleWhile(HloInstruction* xla_while) = 0; diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h index a5fe120598..a1d7acf904 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h @@ -60,14 +60,10 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { return DefaultAction(hlo); } - Status HandleClamp(HloInstruction* clamp, HloInstruction* /*min*/, - HloInstruction* /*arg*/, - HloInstruction* /*max*/) override { + Status HandleClamp(HloInstruction* clamp) override { return DefaultAction(clamp); } - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice /*operands*/) override { + Status HandleConcatenate(HloInstruction* concatenate) override { return DefaultAction(concatenate); } Status HandleConvert(HloInstruction* convert) override { @@ -76,30 +72,20 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { Status HandleCopy(HloInstruction* copy) override { return DefaultAction(copy); } - Status HandleSelect(HloInstruction* select, HloInstruction* /*pred*/, - HloInstruction* /*on_true*/, - HloInstruction* /*on_false*/) override { + Status HandleSelect(HloInstruction* select) override { return DefaultAction(select); } - Status HandleDot(HloInstruction* dot, HloInstruction* /*lhs*/, - HloInstruction* /*rhs*/) override { - return DefaultAction(dot); - } - Status HandleConvolution(HloInstruction* convolution, HloInstruction* /*lhs*/, - HloInstruction* /*rhs*/, - const Window& /*window*/) override { + Status HandleDot(HloInstruction* dot) override { return DefaultAction(dot); } + Status HandleConvolution(HloInstruction* convolution) override { return DefaultAction(convolution); } Status HandleCrossReplicaSum(HloInstruction* crs) override { return DefaultAction(crs); } - Status HandleCompare(HloInstruction* compare, HloOpcode /*opcode*/, - HloInstruction* /*lhs*/, - HloInstruction* /*rhs*/) override { + Status HandleCompare(HloInstruction* compare) override { return DefaultAction(compare); } - Status HandleRng(HloInstruction* random, - RandomDistribution /*distribution*/) override { + Status HandleRng(HloInstruction* random) override { return DefaultAction(random); } Status HandleInfeed(HloInstruction* infeed) override { @@ -108,20 +94,16 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { Status HandleOutfeed(HloInstruction* outfeed) override { return DefaultAction(outfeed); } - Status HandleReverse(HloInstruction* reverse, - HloInstruction* /*operand*/) override { + Status HandleReverse(HloInstruction* reverse) override { return DefaultAction(reverse); } - Status HandleSort(HloInstruction* sort, - HloInstruction* /*operand*/) override { + Status HandleSort(HloInstruction* sort) override { return DefaultAction(sort); } - Status HandleConstant(HloInstruction* constant, - const Literal& /*literal*/) override { + Status HandleConstant(HloInstruction* constant) override { return DefaultAction(constant); } - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* /*operand*/) override { + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override { return DefaultAction(get_tuple_element); } Status HandleParameter(HloInstruction* parameter) override { @@ -133,50 +115,27 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { Status HandleCall(HloInstruction* call) override { return DefaultAction(call); } - Status HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice /*operands*/, - tensorflow::StringPiece /*custom_call_target*/) override { + Status HandleCustomCall(HloInstruction* custom_call) override { return DefaultAction(custom_call); } - Status HandleSlice(HloInstruction* slice, - HloInstruction* /*operand*/) override { + Status HandleSlice(HloInstruction* slice) override { return DefaultAction(slice); } - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* /*operand*/, - HloInstruction* /*start_indices*/) override { + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override { return DefaultAction(dynamic_slice); } - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* /*operand*/, - HloInstruction* /*update*/, - HloInstruction* /*start_indices*/) override { + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override { return DefaultAction(dynamic_update_slice); } - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice /*operands*/) override { + Status HandleTuple(HloInstruction* tuple) override { return DefaultAction(tuple); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice /*operands*/, - HloComputation* /*function*/, - tensorflow::gtl::ArraySlice /*static_operands*/) - override { - return DefaultAction(map); - } - Status HandleReduce(HloInstruction* reduce, HloInstruction* /*arg*/, - HloInstruction* /*init_value*/, - tensorflow::gtl::ArraySlice /*dimensions*/, - HloComputation* /*function*/) override { + Status HandleMap(HloInstruction* map) override { return DefaultAction(map); } + Status HandleReduce(HloInstruction* reduce) override { return DefaultAction(reduce); } - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* /*operand*/, - const Window& /*window*/, - HloComputation* /*function*/) override { + Status HandleReduceWindow(HloInstruction* reduce_window) override { return DefaultAction(reduce_window); } Status HandleSelectAndScatter(HloInstruction* select_and_scatter) override { diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc index 23765e05e8..57a3f713e3 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc @@ -77,8 +77,8 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) { .MakeElementGenerator(hlo, operand_to_generator)); } -Status IrEmitter::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status IrEmitter::HandleConstant(HloInstruction* constant) { + const Literal& literal = constant->literal(); llvm::Constant* initializer = llvm_ir::ConvertLiteralToIrConstant(literal, module_); llvm::GlobalVariable* global_for_const = new llvm::GlobalVariable( @@ -106,8 +106,8 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } -Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) { + auto operand = get_tuple_element->operand(0); CHECK(bindings_.BoundToIrValue(*operand)); bindings_.BindHloToIrValue( *get_tuple_element, @@ -119,25 +119,22 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, return Status::OK(); } -Status IrEmitter::HandleSort(HloInstruction* sort, - HloInstruction* operand_instruction) { +Status IrEmitter::HandleSort(HloInstruction*) { // TODO(b/26783907): Implement sort on GPU. return Unimplemented("sort"); } -Status IrEmitter::HandleSend(HloInstruction* send) { +Status IrEmitter::HandleSend(HloInstruction*) { return Unimplemented("Send is not implemented on GPU"); } -Status IrEmitter::HandleRecv(HloInstruction* recv) { +Status IrEmitter::HandleRecv(HloInstruction*) { return Unimplemented("Recv is not implemented on GPU"); } -Status IrEmitter::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitter::HandleTuple(HloInstruction* tuple) { std::vector base_ptrs; - for (const HloInstruction* operand : operands) { + for (const HloInstruction* operand : tuple->operands()) { base_ptrs.push_back(GetBasePointer(*operand)); } llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_, module_); @@ -321,9 +318,10 @@ Status IrEmitter::EmitAtomicOperationForNestedComputation( return Status::OK(); } -Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { +Status IrEmitter::HandleSelect(HloInstruction* select) { + auto pred = select->operand(0); + auto on_true = select->operand(1); + auto on_false = select->operand(2); TF_RET_CHECK(pred->shape().element_type() == PRED); if (ShapeUtil::IsTuple(select->shape())) { @@ -339,9 +337,9 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, return IrEmitter::DefaultAction(select); } -Status IrEmitter::HandleDot(HloInstruction* dot, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) { +Status IrEmitter::HandleDot(HloInstruction* dot) { + auto lhs_instruction = dot->operand(0); + auto rhs_instruction = dot->operand(1); const llvm_ir::IrArray& target_array = GetIrArray(*dot); const llvm_ir::IrArray& lhs_array = GetIrArray(*lhs_instruction); const llvm_ir::IrArray& rhs_array = GetIrArray(*rhs_instruction); @@ -498,10 +496,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot, return Status::OK(); } -Status IrEmitter::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction, - const Window& window) { +Status IrEmitter::HandleConvolution(HloInstruction* convolution) { if (ShapeUtil::HasZeroElements(convolution->shape())) { // Emit no code for an empty output. return Status::OK(); @@ -521,10 +516,11 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) { return Status::OK(); } -Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) { +Status IrEmitter::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->operand(0); + auto init_value = reduce->operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); return EmitTargetElementLoop( *reduce, [=](const llvm_ir::IrArray::Index& index) -> StatusOr { @@ -601,23 +597,19 @@ Status IrEmitter::HandleCall(HloInstruction* call) { GetBasePointer(*call)); } -Status IrEmitter::HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) { +Status IrEmitter::HandleCustomCall(HloInstruction*) { return Unimplemented("custom-call"); } -Status IrEmitter::HandleInfeed(HloInstruction* infeed) { +Status IrEmitter::HandleInfeed(HloInstruction*) { return Unimplemented("Infeed is not supported on GPU (b/30467474)."); } -Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) { +Status IrEmitter::HandleOutfeed(HloInstruction*) { return Unimplemented("Outfeed is not supported on GPU (b/34359662)."); } -Status IrEmitter::HandleRng(HloInstruction* random, - RandomDistribution /*distribution*/) { +Status IrEmitter::HandleRng(HloInstruction* random) { ElementalIrEmitter::HloToElementGeneratorMap operand_to_generator; for (const HloInstruction* operand : random->operands()) { operand_to_generator[operand] = [=](const llvm_ir::IrArray::Index& index) { diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h index 90f40639d5..263992d925 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h @@ -74,39 +74,25 @@ class IrEmitter : public DfsHloVisitorWithDefault { // The following methods implement the DfsHloVisitorWithDefault interface. Status DefaultAction(HloInstruction* hlo) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; Status HandleBitcast(HloInstruction* bitcast) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleDot(HloInstruction* dot) override; + Status HandleConvolution(HloInstruction* convolution) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; Status HandleInfeed(HloInstruction* infeed) override; Status HandleOutfeed(HloInstruction* outfeed) override; - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override; + Status HandleSort(HloInstruction* sort) override; Status HandleSend(HloInstruction* send) override; Status HandleRecv(HloInstruction* recv) override; Status HandleParameter(HloInstruction* parameter) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleReduce(HloInstruction* reduce) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleSelect(HloInstruction* select) override; Status HandleFusion(HloInstruction* fusion) override; Status HandleCall(HloInstruction* call) override; - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override; - Status HandleRng(HloInstruction* random, - RandomDistribution /*distribution*/) override; + Status HandleCustomCall(HloInstruction* custom_call) override; + Status HandleRng(HloInstruction* random) override; Status FinishVisit(HloInstruction* root) override { return Status::OK(); } @@ -233,28 +219,17 @@ class IrEmitterUnnested : public IrEmitter { // IrEmitterUnnested handles the following instructions differently from // IrEmitter. Status HandleCopy(HloInstruction* copy) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) override; + Status HandleConvolution(HloInstruction* convolution) override; + Status HandleDot(HloInstruction* dot) override; Status HandleFusion(HloInstruction* fusion) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleReduce(HloInstruction* reduce) override; Status HandleSelectAndScatter(HloInstruction* instruction) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; + Status HandleTuple(HloInstruction* tuple) override; Status HandleWhile(HloInstruction* xla_while) override; Status HandleInfeed(HloInstruction* xla_infeed) override; - Status HandleRng(HloInstruction* random, - RandomDistribution distribution) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleRng(HloInstruction* random) override; + Status HandleSelect(HloInstruction* select) override; Status EmitTargetElementLoop( const HloInstruction& hlo, diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 1c7e18304d..7b4662fc80 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -245,28 +245,22 @@ Status IrEmitterUnnested::DefaultAction(HloInstruction* hlo) { return IrEmitter::DefaultAction(hlo); } -Status IrEmitterUnnested::HandleDot(HloInstruction* dot, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) { +Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { if (ImplementedAsGemm(*dot)) { thunk_sequence_->emplace_back(BuildGemmThunk(dot)); return Status::OK(); } thunk_sequence_->emplace_back(BuildKernelThunk(dot)); - return IrEmitter::HandleDot(dot, lhs_instruction, rhs_instruction); + return IrEmitter::HandleDot(dot); } -Status IrEmitterUnnested::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction, - const Window& window) { +Status IrEmitterUnnested::HandleConvolution(HloInstruction* convolution) { if (ImplementedAsDnnConvolution(*convolution)) { thunk_sequence_->emplace_back(BuildConvolutionThunk(convolution)); return Status::OK(); } thunk_sequence_->emplace_back(BuildKernelThunk(convolution)); - return IrEmitter::HandleConvolution(convolution, lhs_instruction, - rhs_instruction, window); + return IrEmitter::HandleConvolution(convolution); } Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) { @@ -1234,10 +1228,11 @@ Status IrEmitterUnnested::EmitReductionToVector( } } -Status IrEmitterUnnested::HandleReduce( - HloInstruction* reduce, HloInstruction* input, HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions_to_reduce, - HloComputation* reducer) { +Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) { + auto input = reduce->operand(0); + auto init_value = reduce->operand(1); + tensorflow::gtl::ArraySlice dimensions_to_reduce(reduce->dimensions()); + HloComputation* reducer = reduce->to_apply(); // HandleReduce specializes reduction from a multi-dimensional array to a 1D // array. The specialized version requires an initializer thunk that // initializes the output array to the initial value of the reduce. @@ -1265,13 +1260,11 @@ Status IrEmitterUnnested::HandleReduce( } thunk_sequence_->emplace_back(BuildKernelThunk(reduce)); - return IrEmitter::HandleReduce(reduce, input, init_value, - dimensions_to_reduce, reducer); + return IrEmitter::HandleReduce(reduce); } -Status IrEmitterUnnested::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitterUnnested::HandleTuple(HloInstruction* tuple) { + tensorflow::gtl::ArraySlice operands(tuple->operands()); bool all_tuple_elements_have_buffer = std::all_of( operands.begin(), operands.end(), [this](HloInstruction* tuple_element) { return ir_emitter_context_->buffer_assignment().HasTopLevelAllocation( @@ -1296,11 +1289,10 @@ Status IrEmitterUnnested::HandleTuple( return Status::OK(); } thunk_sequence_->emplace_back(BuildKernelThunk(tuple)); - return IrEmitter::HandleTuple(tuple, operands); + return IrEmitter::HandleTuple(tuple); } -Status IrEmitterUnnested::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { +Status IrEmitterUnnested::HandleGetTupleElement(HloInstruction*) { // GetTupleElement IR is emitted in the IR context of the user instruction, // and so we do not build a kernel for GetTupleElement instructions. return Status::OK(); @@ -1525,18 +1517,14 @@ Status IrEmitterUnnested::HandleWhile(HloInstruction* xla_while) { return Status::OK(); } -Status IrEmitterUnnested::HandleRng(HloInstruction* random, - RandomDistribution distribution) { +Status IrEmitterUnnested::HandleRng(HloInstruction* random) { thunk_sequence_->push_back(BuildKernelThunk(random)); - return IrEmitter::HandleRng(random, distribution); + return IrEmitter::HandleRng(random); } -Status IrEmitterUnnested::HandleSelect(HloInstruction* select, - HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { +Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { thunk_sequence_->push_back(BuildKernelThunk(select)); - return IrEmitter::HandleSelect(select, pred, on_true, on_false); + return IrEmitter::HandleSelect(select); } Status IrEmitterUnnested::HandleInfeed(HloInstruction* infeed) { diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc index ca99fd6de8..ab018c4cf2 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc @@ -126,16 +126,11 @@ Status HloCostAnalysis::HandleElementwiseBinary(HloInstruction* hlo) { return HandleElementwiseOp(hlo); } -Status HloCostAnalysis::HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, - HloInstruction* rhs) { +Status HloCostAnalysis::HandleCompare(HloInstruction* compare) { return HandleElementwiseOp(compare); } -Status HloCostAnalysis::HandleClamp(HloInstruction* clamp, - HloInstruction* min_instruction, - HloInstruction* arg_instruction, - HloInstruction* max_instruction) { +Status HloCostAnalysis::HandleClamp(HloInstruction* clamp) { return HandleElementwiseOp(clamp); } @@ -143,57 +138,38 @@ Status HloCostAnalysis::HandleReducePrecision(HloInstruction* hlo) { return HandleElementwiseOp(hlo); } -Status HloCostAnalysis::HandleParameter(HloInstruction* parameter) { +Status HloCostAnalysis::HandleParameter(HloInstruction*) { current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status HloCostAnalysis::HandleConstant(HloInstruction*) { current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status HloCostAnalysis::HandleGetTupleElement(HloInstruction*) { // GetTupleElement forwards a pointer and does not touch each element in the // output. current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleSelect(HloInstruction* select, - HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { - return Status::OK(); -} +Status HloCostAnalysis::HandleSelect(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleReverse(HloInstruction* reverse, - HloInstruction* operand_instruction) { - return Status::OK(); -} +Status HloCostAnalysis::HandleReverse(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleSlice(HloInstruction* slice, - HloInstruction* operand_instruction) { - return Status::OK(); -} +Status HloCostAnalysis::HandleSlice(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) { +Status HloCostAnalysis::HandleDynamicSlice(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleDynamicUpdateSlice( - HloInstruction* dynamic_update, HloInstruction* operand, - HloInstruction* update, HloInstruction* start_indices) { +Status HloCostAnalysis::HandleDynamicUpdateSlice(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status HloCostAnalysis::HandleTuple(HloInstruction* tuple) { // The tuple instruction only gathers pointers from inputs (it doesn't iterate // through them). The memory touched is then only the size of the output // index table of the tuple. @@ -202,9 +178,7 @@ Status HloCostAnalysis::HandleTuple( return Status::OK(); } -Status HloCostAnalysis::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { +Status HloCostAnalysis::HandleConcatenate(HloInstruction*) { return Status::OK(); } @@ -212,15 +186,11 @@ Status HloCostAnalysis::HandleConvert(HloInstruction* convert) { return HandleElementwiseOp(convert); } -Status HloCostAnalysis::HandleCopy(HloInstruction* copy) { - return Status::OK(); -} +Status HloCostAnalysis::HandleCopy(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleDot(HloInstruction* dot, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) { - const Shape& lhs_shape = lhs_instruction->shape(); - const Shape& rhs_shape = rhs_instruction->shape(); +Status HloCostAnalysis::HandleDot(HloInstruction* dot) { + const Shape& lhs_shape = dot->operand(0)->shape(); + const Shape& rhs_shape = dot->operand(1)->shape(); // Count of elements along the reduction dimension (last dimension for the // rhs). int64 reduction_width = lhs_shape.dimensions(ShapeUtil::Rank(lhs_shape) - 1); @@ -240,21 +210,14 @@ Status HloCostAnalysis::HandleDot(HloInstruction* dot, return Status::OK(); } -Status HloCostAnalysis::HandleInfeed(HloInstruction* infeed) { - return Status::OK(); -} +Status HloCostAnalysis::HandleInfeed(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleOutfeed(HloInstruction* outfeed) { - return Status::OK(); -} +Status HloCostAnalysis::HandleOutfeed(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleMap( - HloInstruction* map, tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice /*static_operands*/) { +Status HloCostAnalysis::HandleMap(HloInstruction* map) { // Compute properties of the mapped function. TF_ASSIGN_OR_RETURN(const Properties sub_properties, - ProcessSubcomputation(function)); + ProcessSubcomputation(map->to_apply())); // Compute the cost of all elements for this Map operation. const int64 element_count = ShapeUtil::ElementsIn(map->shape()); @@ -266,9 +229,9 @@ Status HloCostAnalysis::HandleMap( return Status::OK(); } -Status HloCostAnalysis::HandleReduce( - HloInstruction* reduce, HloInstruction* arg, HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, HloComputation* function) { +Status HloCostAnalysis::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->operand(0); + HloComputation* function = reduce->to_apply(); // Compute the cost of the user function. TF_ASSIGN_OR_RETURN(const Properties sub_properties, ProcessSubcomputation(function)); @@ -284,10 +247,9 @@ Status HloCostAnalysis::HandleReduce( return Status::OK(); } -Status HloCostAnalysis::HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, - const Window& window, - HloComputation* function) { +Status HloCostAnalysis::HandleReduceWindow(HloInstruction* reduce_window) { + const Window& window = reduce_window->window(); + auto function = reduce_window->to_apply(); // Compute the properties of the reduction function. TF_ASSIGN_OR_RETURN(const Properties sub_properties, ProcessSubcomputation(function)); @@ -342,55 +304,45 @@ Status HloCostAnalysis::HandleSelectAndScatter(HloInstruction* instruction) { return Status::OK(); } -Status HloCostAnalysis::HandleBitcast(HloInstruction* bitcast) { +Status HloCostAnalysis::HandleBitcast(HloInstruction*) { // A bitcast does no computation and touches no memory. current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleBroadcast(HloInstruction* broadcast) { +Status HloCostAnalysis::HandleBroadcast(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandlePad(HloInstruction* pad) { return Status::OK(); } +Status HloCostAnalysis::HandlePad(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleSend(HloInstruction* send) { - return Status::OK(); -} +Status HloCostAnalysis::HandleSend(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleRecv(HloInstruction* recv) { - return Status::OK(); -} +Status HloCostAnalysis::HandleRecv(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleReshape(HloInstruction* reshape) { - return Status::OK(); -} +Status HloCostAnalysis::HandleReshape(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleBatchNormTraining( - HloInstruction* batch_norm_training) { +Status HloCostAnalysis::HandleBatchNormTraining(HloInstruction*) { // TODO(b/62294698): Implement cost analysis for batch-norm-training. return Status::OK(); } -Status HloCostAnalysis::HandleBatchNormInference( - HloInstruction* batch_norm_inference) { +Status HloCostAnalysis::HandleBatchNormInference(HloInstruction*) { // TODO(b/62294698): Implement cost analysis for batch-norm-inference. return Status::OK(); } -Status HloCostAnalysis::HandleBatchNormGrad(HloInstruction* batch_norm_grad) { +Status HloCostAnalysis::HandleBatchNormGrad(HloInstruction*) { // TODO(b/62294698): Implement cost analysis for batch-norm-grad. return Status::OK(); } -Status HloCostAnalysis::HandleTranspose(HloInstruction* transpose) { +Status HloCostAnalysis::HandleTranspose(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction, - const Window& window) { +Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution) { + auto rhs_instruction = convolution->operand(1); const auto& dnums = convolution->convolution_dimension_numbers(); const int64 output_features = convolution->shape().dimensions(dnums.output_feature_dimension()); @@ -417,8 +369,7 @@ Status HloCostAnalysis::HandleCrossReplicaSum(HloInstruction* crs) { return Status::OK(); } -Status HloCostAnalysis::HandleRng(HloInstruction* random, - RandomDistribution distribution) { +Status HloCostAnalysis::HandleRng(HloInstruction* random) { // TODO(b/26346211): Implement better estimates for the RNG cost, since the // cost changes with the implementation and the distribution. For now, assume // the cost of each RNG is same as a transcendental operation. @@ -462,18 +413,14 @@ Status HloCostAnalysis::HandleCall(HloInstruction* call) { return Status::OK(); } -Status HloCostAnalysis::HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) { +Status HloCostAnalysis::HandleCustomCall(HloInstruction*) { return Unimplemented("Custom-call is not implemented for HLO cost analysis."); } -Status HloCostAnalysis::HandleSort(HloInstruction* sort, - HloInstruction* operand_instruction) { +Status HloCostAnalysis::HandleSort(HloInstruction* sort) { // This assumes a comparison based N*log(N) algorithm. As for all ops, the // actual properties of the op depend on the backend implementation. - int64 elements = ShapeUtil::ElementsIn(operand_instruction->shape()); + int64 elements = ShapeUtil::ElementsIn(sort->operand(0)->shape()); current_properties_[kFlopsKey] = elements * tensorflow::Log2Ceiling(elements); return Status::OK(); } @@ -502,9 +449,7 @@ Status HloCostAnalysis::HandleWhile(HloInstruction* xla_while) { return Status::OK(); } -Status HloCostAnalysis::FinishVisit(HloInstruction* root) { - return Status::OK(); -} +Status HloCostAnalysis::FinishVisit(HloInstruction*) { return Status::OK(); } float HloCostAnalysis::flop_count() const { return GetProperty(kFlopsKey, properties_sum_); diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h index d71c2eccee..93b1b3eb20 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h @@ -51,70 +51,41 @@ class HloCostAnalysis : public DfsHloVisitor { Status HandleElementwiseUnary(HloInstruction* hlo) override; Status HandleElementwiseBinary(HloInstruction* hlo) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; - Status HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) override; - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override; + Status HandleConstant(HloInstruction* constant) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleSelect(HloInstruction* select) override; + Status HandleCompare(HloInstruction* compare) override; + Status HandleClamp(HloInstruction* clamp) override; Status HandleReducePrecision(HloInstruction* hlo) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; Status HandleSend(HloInstruction* send) override; Status HandleRecv(HloInstruction* recv) override; Status HandleConvert(HloInstruction* convert) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleDot(HloInstruction* dot) override; + Status HandleConvolution(HloInstruction* convolution) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; Status HandleInfeed(HloInstruction* infeed) override; Status HandleOutfeed(HloInstruction* outfeed) override; - Status HandleRng(HloInstruction* random, - RandomDistribution distribution) override; - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override; - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override; + Status HandleRng(HloInstruction* random) override; + Status HandleReverse(HloInstruction* reverse) override; + Status HandleSort(HloInstruction* sort) override; Status HandleParameter(HloInstruction* parameter) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function_handle) override; + Status HandleReduce(HloInstruction* reduce) override; Status HandleBatchNormTraining(HloInstruction* batch_norm_training) override; Status HandleBatchNormInference( HloInstruction* batch_norm_inference) override; Status HandleBatchNormGrad(HloInstruction* batch_norm_grad) override; Status HandleFusion(HloInstruction* fusion) override; Status HandleCall(HloInstruction* call) override; - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override; - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override; - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) override; - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override; - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override; + Status HandleCustomCall(HloInstruction* custom_call) override; + Status HandleSlice(HloInstruction* slice) override; + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override; + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleMap(HloInstruction* map) override; + Status HandleReduceWindow(HloInstruction* reduce_window) override; Status HandleSelectAndScatter(HloInstruction* instruction) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleBroadcast(HloInstruction* broadcast) override; diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index f4a2c3d0e8..88b77ccdd0 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -183,7 +183,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { template ::value>::type* = nullptr> - Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { + Status HandleAbs(HloInstruction* abs) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[abs], ElementWiseUnaryOp(abs, [](NativeT elem_operand) { return elem_operand; @@ -195,7 +195,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename NativeT, typename std::enable_if::value || is_complex_t::value>::type* = nullptr> - Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { + Status HandleAbs(HloInstruction* abs) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[abs], ElementWiseUnaryOp(abs, [](NativeT elem_operand) { return std::abs(elem_operand); @@ -203,8 +203,8 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleAbs(HloInstruction* abs, HloInstruction* operand) override { - return HandleAbs(abs, operand); + Status HandleAbs(HloInstruction* abs) override { + return HandleAbs(abs); } template < @@ -277,7 +277,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return InvalidArgument("Unsupported type for Ceil"); } - Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) override { + Status HandleCeil(HloInstruction* ceil) override { return HandleCeil(ceil); } @@ -297,7 +297,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleExp(HloInstruction* exp, HloInstruction* operand) override { + Status HandleExp(HloInstruction* exp) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[exp], ElementWiseUnaryOp(exp, [](ReturnT elem_operand) { return std::exp(elem_operand); @@ -323,11 +323,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return InvalidArgument("Unsupported type for Floor"); } - Status HandleFloor(HloInstruction* floor, HloInstruction* operand) override { + Status HandleFloor(HloInstruction* floor) override { return HandleFloor(floor); } - Status HandleLog(HloInstruction* log, HloInstruction* operand) override { + Status HandleLog(HloInstruction* log) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[log], ElementWiseUnaryOp(log, [](ReturnT elem_operand) { return std::log(elem_operand); @@ -353,12 +353,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return InvalidArgument("Unsupported type for Not"); } - Status HandleNot(HloInstruction* not_, HloInstruction* operand) override { + Status HandleNot(HloInstruction* not_) override { return HandleNot(not_); } - Status HandleNegate(HloInstruction* negate, - HloInstruction* operand) override { + Status HandleNegate(HloInstruction* negate) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[negate], ElementWiseUnaryOp(negate, [](ReturnT elem_operand) { return -elem_operand; @@ -391,11 +390,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleSign(HloInstruction* sign, HloInstruction* operand) override { + Status HandleSign(HloInstruction* sign) override { return HandleSign(sign); } - Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) override { + Status HandleTanh(HloInstruction* tanh) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[tanh], ElementWiseUnaryOp(tanh, [](ReturnT elem_operand) { return std::tanh(elem_operand); @@ -403,8 +402,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleMultiply(HloInstruction* multiply) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[multiply], ElementWiseBinaryOp(multiply, [](ReturnT lhs_elem, ReturnT rhs_elem) { @@ -413,8 +411,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleSubtract(HloInstruction* subtract) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[subtract], ElementWiseBinaryOp(subtract, [](ReturnT lhs_elem, ReturnT rhs_elem) { @@ -423,8 +420,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleAdd(HloInstruction* add) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[add], ElementWiseBinaryOp(add, [](ReturnT lhs_elem, ReturnT rhs_elem) { @@ -433,8 +429,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleDivide(HloInstruction* divide) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[divide], ElementWiseBinaryOp(divide, [](ReturnT lhs_elem, ReturnT rhs_elem) { @@ -489,8 +484,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return HandleMinimum(minimum); } - Status HandlePower(HloInstruction* power, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandlePower(HloInstruction* power) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[power], ElementWiseBinaryOp(power, [](ReturnT lhs_el, ReturnT rhs_el) { @@ -518,8 +512,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return InvalidArgument("Unsupported type for Remainder"); } - Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleRemainder(HloInstruction* remainder) override { return HandleRemainder(remainder); } @@ -542,8 +535,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return InvalidArgument("Unsupported type for And"); } - Status HandleAnd(HloInstruction* and_, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleAnd(HloInstruction* and_) override { return HandleAnd(and_); } @@ -566,8 +558,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return InvalidArgument("Unsupported type for Or"); } - Status HandleOr(HloInstruction* or_, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleOr(HloInstruction* or_) override { return HandleOr(or_); } @@ -575,8 +566,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if< std::is_integral::value && !std::is_same::value>::type* = nullptr> - Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftLeft(HloInstruction* shl) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[shl], ElementWiseBinaryOp(shl, [](NativeT lhs_elem, NativeT rhs_elem) { @@ -589,21 +579,18 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if::value || std::is_same::value>::type* = nullptr> - Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftLeft(HloInstruction*) { return InvalidArgument("Unsupported type for ShiftLeft"); } - Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs, - HloInstruction* rhs) override { - return HandleShiftLeft(shl, lhs, rhs); + Status HandleShiftLeft(HloInstruction* shl) override { + return HandleShiftLeft(shl); } template ::value && !std::is_same::value>::type* = nullptr> - Status HandleShiftRightArithmetic(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightArithmetic(HloInstruction* shr) { typedef typename std::make_signed::type SignedT; TF_ASSIGN_OR_RETURN( parent_->evaluated_[shr], @@ -618,22 +605,19 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if::value || std::is_same::value>::type* = nullptr> - Status HandleShiftRightArithmetic(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightArithmetic(HloInstruction*) { return InvalidArgument("Unsupported type for ShiftRightArithmetic"); } - Status HandleShiftRightArithmetic(HloInstruction* shra, HloInstruction* lhs, - HloInstruction* rhs) override { - return HandleShiftRightArithmetic(shra, lhs, rhs); + Status HandleShiftRightArithmetic(HloInstruction* shra) override { + return HandleShiftRightArithmetic(shra); } template ::value && !std::is_same::value>::type* = nullptr> - Status HandleShiftRightLogical(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightLogical(HloInstruction* shr) { typedef typename std::make_unsigned::type UnsignedT; TF_ASSIGN_OR_RETURN( parent_->evaluated_[shr], @@ -648,21 +632,18 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if::value || std::is_same::value>::type* = nullptr> - Status HandleShiftRightLogical(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightLogical(HloInstruction*) { return InvalidArgument("Unsupported type for ShiftRightLogical"); } - Status HandleShiftRightLogical(HloInstruction* shrl, HloInstruction* lhs, - HloInstruction* rhs) override { - return HandleShiftRightLogical(shrl, lhs, rhs); + Status HandleShiftRightLogical(HloInstruction* shrl) override { + return HandleShiftRightLogical(shrl); } template < typename NativeT, typename std::enable_if::value>::type* = nullptr> - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) { + Status HandleClamp(HloInstruction* clamp) { std::function clamp_op = [](ReturnT low, ReturnT high, ReturnT value) { return std::fmax(low, std::fmin(value, high)); @@ -675,19 +656,15 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { template < typename NativeT, typename std::enable_if::value>::type* = nullptr> - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) { + Status HandleClamp(HloInstruction*) { return InvalidArgument("Unsupported type for Clamp"); } - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override { - return HandleClamp(clamp, min, arg, max); + Status HandleClamp(HloInstruction* clamp) override { + return HandleClamp(clamp); } - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override { + Status HandleSelect(HloInstruction* select) override { CHECK(!ShapeUtil::IsTuple(select->shape())); std::function select_op = [](bool pred, ReturnT on_true, ReturnT on_false) { @@ -701,11 +678,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override { + Status HandleReverse(HloInstruction* reverse) override { const auto result_shape = reverse->shape(); const auto reverse_dimensions = reverse->dimensions(); + auto operand = reverse->operand(0); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, ShapeInference::InferReverseShape(operand->shape(), reverse_dimensions)); @@ -731,8 +708,10 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleConvolution(HloInstruction* conv, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override { + Status HandleConvolution(HloInstruction* conv) override { + auto lhs = conv->operand(0); + auto rhs = conv->operand(1); + const auto& window = conv->window(); const Shape& result_shape = conv->shape(); const Shape& lhs_shape = lhs->shape(); const Shape& rhs_shape = rhs->shape(); @@ -854,8 +833,9 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleDot(HloInstruction* dot) override { + auto lhs = dot->operand(0); + auto rhs = dot->operand(1); CHECK(ShapeUtil::IsArray(dot->shape())); CHECK(ShapeUtil::IsArray(lhs->shape())); CHECK(ShapeUtil::IsArray(rhs->shape())); @@ -990,9 +970,9 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) override { + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override { + auto operand = dynamic_slice->operand(0); + auto start_indices = dynamic_slice->operand(1); auto result_shape = dynamic_slice->shape(); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, ShapeInference::InferDynamicSliceShape( @@ -1043,10 +1023,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override { + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override { + auto operand = dynamic_update_slice->operand(0); + auto update = dynamic_update_slice->operand(1); + auto start_indices = dynamic_update_slice->operand(2); auto result_shape = dynamic_update_slice->shape(); TF_ASSIGN_OR_RETURN( auto inferred_return_shape, @@ -1099,10 +1080,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override { + Status HandleReduce(HloInstruction* reduce) override { + auto arg = reduce->operand(0); + auto init_value = reduce->operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); TF_RET_CHECK(ShapeUtil::Rank(reduce->shape()) == ShapeUtil::Rank(arg->shape()) - dimensions.size()); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, @@ -1187,9 +1169,10 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override { + Status HandleReduceWindow(HloInstruction* reduce_window) override { + auto operand = reduce_window->operand(0); + const Window& window = reduce_window->window(); + HloComputation* function = reduce_window->to_apply(); TF_ASSIGN_OR_RETURN( auto inferred_return_shape, ShapeInference::InferReduceWindowShape( @@ -1274,7 +1257,8 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override { + Status HandleSlice(HloInstruction* slice) override { + auto operand = slice->operand(0); const Shape& shape = slice->shape(); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, ShapeInference::InferSliceShape( @@ -1603,10 +1587,7 @@ Status HloEvaluator::HandleParameter(HloInstruction* parameter) { return Status::OK(); } -Status HloEvaluator::HandleConstant(HloInstruction* constant, - const Literal& literal) { - return Status::OK(); -} +Status HloEvaluator::HandleConstant(HloInstruction*) { return Status::OK(); } Status HloEvaluator::HandleReshape(HloInstruction* reshape) { TF_ASSIGN_OR_RETURN( @@ -1622,9 +1603,9 @@ Status HloEvaluator::HandleTranspose(HloInstruction* transpose) { return Status::OK(); } -Status HloEvaluator::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { +Status HloEvaluator::HandleConcatenate(HloInstruction* concatenate) { + tensorflow::gtl::ArraySlice operands( + concatenate->operands()); // The result concatenate dimension is going to be the sum of all concatenate // dimensions of the operands taking part of the operation. const Shape& reference_shape = operands[0]->shape(); @@ -1664,8 +1645,8 @@ Status HloEvaluator::HandleConcatenate( return Status::OK(); } -Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite, - HloInstruction* operand) { +Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite) { + auto operand = is_finite->operand(0); if (!ShapeUtil::ElementIsFloating(operand->shape())) { return InvalidArgument( "expected element type in shape to be float for IsFinite op, got: %s", @@ -1699,8 +1680,10 @@ Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite, return Status::OK(); } -Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) { +Status HloEvaluator::HandleCompare(HloInstruction* compare) { + HloOpcode opcode = compare->opcode(); + auto lhs = compare->operand(0); + auto rhs = compare->operand(1); // TODO(b/35950897, b/27796129): add DCHECK back once implicit broadcast is // removed. if (!(ShapeUtil::SameDimensions(compare->shape(), rhs->shape()) && @@ -1784,11 +1767,9 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode, return Status::OK(); } -Status HloEvaluator::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status HloEvaluator::HandleTuple(HloInstruction* tuple) { std::vector operand_literals; - for (auto operand : operands) { + for (auto operand : tuple->operands()) { operand_literals.push_back(&GetEvaluatedLiteralFor(operand)); } @@ -1796,11 +1777,11 @@ Status HloEvaluator::HandleTuple( return Status::OK(); } -Status HloEvaluator::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status HloEvaluator::HandleGetTupleElement(HloInstruction* get_tuple_element) { const auto result_shape = get_tuple_element->shape(); const int64 index = get_tuple_element->tuple_index(); + auto operand = get_tuple_element->operand(0); TF_ASSIGN_OR_RETURN( auto inferred_return_shape, ShapeInference::InferGetTupleElementShape(operand->shape(), index)); diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h index a9cecb11be..67b6e215fc 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.h +++ b/tensorflow/compiler/xla/service/hlo_evaluator.h @@ -120,28 +120,20 @@ class HloEvaluator : public DfsHloVisitorWithDefault { // Status HandleParameter(HloInstruction* parameter) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; Status HandleReshape(HloInstruction* reshape) override; Status HandleTranspose(HloInstruction* transpose) override; - Status HandleIsFinite(HloInstruction* is_finite, - HloInstruction* operand) override; + Status HandleIsFinite(HloInstruction* is_finite) override; - Status HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; + Status HandleCompare(HloInstruction* compare) override; + Status HandleTuple(HloInstruction* tuple) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleCopy(HloInstruction* copy) override; diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 4af52717bb..1de4c4a115 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2131,7 +2131,7 @@ HloInstruction::HloInstruction(HloOpcode opcode, const Shape& shape) Status HloInstruction::Visit(DfsHloVisitor* visitor) { switch (opcode_) { case HloOpcode::kAbs: - return visitor->HandleAbs(this, operands_[0]); + return visitor->HandleAbs(this); case HloOpcode::kAtan2: return visitor->HandleAtan2(this, operands_[0], operands_[1]); case HloOpcode::kRoundNearestAfz: @@ -2143,11 +2143,11 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kBatchNormGrad: return visitor->HandleBatchNormGrad(this); case HloOpcode::kSign: - return visitor->HandleSign(this, operands_[0]); + return visitor->HandleSign(this); case HloOpcode::kConstant: - return visitor->HandleConstant(this, *literal_); + return visitor->HandleConstant(this); case HloOpcode::kGetTupleElement: - return visitor->HandleGetTupleElement(this, operands_[0]); + return visitor->HandleGetTupleElement(this); case HloOpcode::kParameter: return visitor->HandleParameter(this); case HloOpcode::kEq: @@ -2156,91 +2156,85 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kLe: case HloOpcode::kLt: case HloOpcode::kNe: - return visitor->HandleCompare(this, opcode_, operands_[0], operands_[1]); + return visitor->HandleCompare(this); case HloOpcode::kComplex: - return visitor->HandleComplex(this, operands_[0], operands_[1]); + return visitor->HandleComplex(this); case HloOpcode::kAdd: - return visitor->HandleAdd(this, operands_[0], operands_[1]); + return visitor->HandleAdd(this); case HloOpcode::kDivide: - return visitor->HandleDivide(this, operands_[0], operands_[1]); + return visitor->HandleDivide(this); case HloOpcode::kSubtract: - return visitor->HandleSubtract(this, operands_[0], operands_[1]); + return visitor->HandleSubtract(this); case HloOpcode::kMaximum: return visitor->HandleMaximum(this); case HloOpcode::kMinimum: return visitor->HandleMinimum(this); case HloOpcode::kAnd: - return visitor->HandleAnd(this, operands_[0], operands_[1]); + return visitor->HandleAnd(this); case HloOpcode::kOr: - return visitor->HandleOr(this, operands_[0], operands_[1]); + return visitor->HandleOr(this); case HloOpcode::kShiftLeft: - return visitor->HandleShiftLeft(this, operands_[0], operands_[1]); + return visitor->HandleShiftLeft(this); case HloOpcode::kShiftRightArithmetic: - return visitor->HandleShiftRightArithmetic(this, operands_[0], - operands_[1]); + return visitor->HandleShiftRightArithmetic(this); case HloOpcode::kShiftRightLogical: - return visitor->HandleShiftRightLogical(this, operands_[0], operands_[1]); + return visitor->HandleShiftRightLogical(this); case HloOpcode::kConcatenate: - return visitor->HandleConcatenate(this, operands_); + return visitor->HandleConcatenate(this); case HloOpcode::kConvert: return visitor->HandleConvert(this); case HloOpcode::kCopy: return visitor->HandleCopy(this); case HloOpcode::kMultiply: - return visitor->HandleMultiply(this, operands_[0], operands_[1]); + return visitor->HandleMultiply(this); case HloOpcode::kDot: - return visitor->HandleDot(this, operands_[0], operands_[1]); + return visitor->HandleDot(this); case HloOpcode::kPower: - return visitor->HandlePower(this, operands_[0], operands_[1]); + return visitor->HandlePower(this); case HloOpcode::kRemainder: - return visitor->HandleRemainder(this, operands_[0], operands_[1]); + return visitor->HandleRemainder(this); case HloOpcode::kSelect: - return visitor->HandleSelect(this, operands_[0], operands_[1], - operands_[2]); + return visitor->HandleSelect(this); case HloOpcode::kConvolution: - return visitor->HandleConvolution(this, operands_[0], operands_[1], - window()); + return visitor->HandleConvolution(this); case HloOpcode::kCrossReplicaSum: return visitor->HandleCrossReplicaSum(this); case HloOpcode::kTuple: - return visitor->HandleTuple(this, operands_); + return visitor->HandleTuple(this); case HloOpcode::kMap: - return visitor->HandleMap(this, operands_, to_apply(), {}); + return visitor->HandleMap(this); case HloOpcode::kClamp: - return visitor->HandleClamp(this, operands_[0], operands_[1], - operands_[2]); + return visitor->HandleClamp(this); case HloOpcode::kReduce: - return visitor->HandleReduce(this, operands_[0], operands_[1], - dimensions_, to_apply()); + return visitor->HandleReduce(this); case HloOpcode::kReduceWindow: - return visitor->HandleReduceWindow(this, operands_[0], window(), - to_apply()); + return visitor->HandleReduceWindow(this); case HloOpcode::kSelectAndScatter: return visitor->HandleSelectAndScatter(this); case HloOpcode::kNegate: - return visitor->HandleNegate(this, operands_[0]); + return visitor->HandleNegate(this); case HloOpcode::kExp: - return visitor->HandleExp(this, operands_[0]); + return visitor->HandleExp(this); case HloOpcode::kFloor: - return visitor->HandleFloor(this, operands_[0]); + return visitor->HandleFloor(this); case HloOpcode::kCeil: - return visitor->HandleCeil(this, operands_[0]); + return visitor->HandleCeil(this); case HloOpcode::kLog: - return visitor->HandleLog(this, operands_[0]); + return visitor->HandleLog(this); case HloOpcode::kTanh: - return visitor->HandleTanh(this, operands_[0]); + return visitor->HandleTanh(this); case HloOpcode::kCos: - return visitor->HandleCos(this, operands_[0]); + return visitor->HandleCos(this); case HloOpcode::kSin: - return visitor->HandleSin(this, operands_[0]); + return visitor->HandleSin(this); case HloOpcode::kReal: - return visitor->HandleReal(this, operands_[0]); + return visitor->HandleReal(this); case HloOpcode::kImag: - return visitor->HandleImag(this, operands_[0]); + return visitor->HandleImag(this); case HloOpcode::kIsFinite: - return visitor->HandleIsFinite(this, operands_[0]); + return visitor->HandleIsFinite(this); case HloOpcode::kNot: - return visitor->HandleNot(this, operands_[0]); + return visitor->HandleNot(this); case HloOpcode::kBitcast: return visitor->HandleBitcast(this); case HloOpcode::kBroadcast: @@ -2252,24 +2246,23 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kTranspose: return visitor->HandleTranspose(this); case HloOpcode::kReverse: - return visitor->HandleReverse(this, operands_[0]); + return visitor->HandleReverse(this); case HloOpcode::kReducePrecision: return visitor->HandleReducePrecision(this); case HloOpcode::kSlice: - return visitor->HandleSlice(this, operands_[0]); + return visitor->HandleSlice(this); case HloOpcode::kDynamicSlice: - return visitor->HandleDynamicSlice(this, operands_[0], operands_[1]); + return visitor->HandleDynamicSlice(this); case HloOpcode::kDynamicUpdateSlice: - return visitor->HandleDynamicUpdateSlice(this, operands_[0], operands_[1], - operands_[2]); + return visitor->HandleDynamicUpdateSlice(this); case HloOpcode::kSort: - return visitor->HandleSort(this, operands_[0]); + return visitor->HandleSort(this); case HloOpcode::kInfeed: return visitor->HandleInfeed(this); case HloOpcode::kOutfeed: return visitor->HandleOutfeed(this); case HloOpcode::kRng: - return visitor->HandleRng(this, distribution_); + return visitor->HandleRng(this); case HloOpcode::kWhile: return visitor->HandleWhile(this); case HloOpcode::kFusion: @@ -2277,7 +2270,7 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kCall: return visitor->HandleCall(this); case HloOpcode::kCustomCall: - return visitor->HandleCustomCall(this, operands_, custom_call_target_); + return visitor->HandleCustomCall(this); case HloOpcode::kSend: return visitor->HandleSend(this); case HloOpcode::kRecv: diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index 9affecae60..4ead64d997 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -59,15 +59,15 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override { + Status HandleConstant(HloInstruction* constant) override { EXPECT_EQ(0, count_.count(constant)); count_[constant] = GetCountsForNode(constant); return Status::OK(); } - Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleAdd(HloInstruction* add) override { + auto lhs = add->operand(0); + auto rhs = add->operand(1); EXPECT_EQ(0, count_.count(add)); EXPECT_GT(count_.count(lhs), 0); EXPECT_GT(count_.count(rhs), 0); @@ -75,32 +75,26 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleNegate(HloInstruction* negate, - HloInstruction* operand) override { + Status HandleNegate(HloInstruction* negate) override { + auto operand = negate->operand(0); EXPECT_EQ(0, count_.count(negate)); EXPECT_GT(count_.count(operand), 0); count_[negate] = GetCountsForNode(negate); return Status::OK(); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* /*function*/, - tensorflow::gtl::ArraySlice /*static_operands*/) - override { + Status HandleMap(HloInstruction* map) override { EXPECT_EQ(0, count_.count(map)); - for (HloInstruction* arg : operands) { + for (HloInstruction* arg : map->operands()) { EXPECT_GT(count_.count(arg), 0); } count_[map] = GetCountsForNode(map); return Status::OK(); } - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override { + Status HandleReduce(HloInstruction* reduce) override { + auto arg = reduce->operand(0); + auto init_value = reduce->operand(1); EXPECT_EQ(0, count_.count(reduce)); EXPECT_GT(count_.count(arg), 0); EXPECT_GT(count_.count(init_value), 0); diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 86ae00971b..c1aa655401 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -40,22 +40,17 @@ class ShapeVerifier : public DfsHloVisitor { return CheckBinaryShape(hlo); } - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override { + Status HandleClamp(HloInstruction* clamp) override { return CheckTernaryShape(clamp); } - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override { + Status HandleSelect(HloInstruction* select) override { return CheckTernaryShape(select); } - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override { + Status HandleConcatenate(HloInstruction* concatenate) override { std::vector operand_shapes; - for (const HloInstruction* operand : operands) { + for (const HloInstruction* operand : concatenate->operands()) { operand_shapes.push_back(&operand->shape()); } return CheckShape( @@ -77,17 +72,17 @@ class ShapeVerifier : public DfsHloVisitor { return CheckUnaryShape(copy); } - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleDot(HloInstruction* dot) override { return CheckBinaryShape(dot); } - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override { - TF_ASSIGN_OR_RETURN(const Shape expected, - ShapeInference::InferConvolveShape( - lhs->shape(), rhs->shape(), window, - convolution->convolution_dimension_numbers())); + Status HandleConvolution(HloInstruction* convolution) override { + TF_ASSIGN_OR_RETURN( + const Shape expected, + ShapeInference::InferConvolveShape( + convolution->operand(0)->shape(), convolution->operand(1)->shape(), + convolution->window(), + convolution->convolution_dimension_numbers())); return CheckShape(convolution, expected); } @@ -104,47 +99,40 @@ class ShapeVerifier : public DfsHloVisitor { reduce_precision->mantissa_bits())); } - Status HandleInfeed(HloInstruction* infeed) override { + Status HandleInfeed(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleOutfeed(HloInstruction* outfeed) override { + Status HandleOutfeed(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleRng(HloInstruction* random, - RandomDistribution distribution) override { + Status HandleRng(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override { + Status HandleReverse(HloInstruction* reverse) override { return CheckShape( reverse, ShapeInference::InferReverseShape(reverse->operand(0)->shape(), reverse->dimensions())); } - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override { + Status HandleSort(HloInstruction* sort) override { return CheckUnaryShape(sort); } - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override { - return CheckShape(constant, literal.shape()); + Status HandleConstant(HloInstruction* constant) override { + return CheckShape(constant, constant->literal().shape()); } - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override { + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override { return CheckShape(get_tuple_element, ShapeInference::InferGetTupleElementShape( get_tuple_element->operand(0)->shape(), get_tuple_element->tuple_index())); } - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override { + Status HandleReduce(HloInstruction* reduce) override { return CheckShape( reduce, ShapeInference::InferReduceShape( @@ -187,11 +175,11 @@ class ShapeVerifier : public DfsHloVisitor { transpose->dimensions())); } - Status HandleParameter(HloInstruction* parameter) override { + Status HandleParameter(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleFusion(HloInstruction* fusion) override { + Status HandleFusion(HloInstruction*) override { return tensorflow::Status::OK(); } @@ -200,32 +188,26 @@ class ShapeVerifier : public DfsHloVisitor { return CheckShape(call, call->to_apply()->ComputeProgramShape().result()); } - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override { + Status HandleCustomCall(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override { + Status HandleSlice(HloInstruction* slice) override { return CheckShape(slice, ShapeInference::InferSliceShape( slice->operand(0)->shape(), slice->slice_starts(), slice->slice_limits(), slice->slice_strides())); } - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) override { + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override { return CheckShape(dynamic_slice, ShapeInference::InferDynamicSliceShape( dynamic_slice->operand(0)->shape(), dynamic_slice->operand(1)->shape(), dynamic_slice->dynamic_slice_sizes())); } - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override { + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override { return CheckShape(dynamic_update_slice, ShapeInference::InferDynamicUpdateSliceShape( dynamic_update_slice->operand(0)->shape(), @@ -233,20 +215,14 @@ class ShapeVerifier : public DfsHloVisitor { dynamic_update_slice->operand(2)->shape())); } - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override { + Status HandleTuple(HloInstruction* tuple) override { return CheckVariadicShape(tuple); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override { + Status HandleMap(HloInstruction* map) override { std::vector operand_shapes; int64 max_operand_rank = 0; - for (const HloInstruction* operand : operands) { + for (const HloInstruction* operand : map->operands()) { operand_shapes.push_back(&operand->shape()); max_operand_rank = std::max(max_operand_rank, ShapeUtil::Rank(operand->shape())); @@ -261,9 +237,7 @@ class ShapeVerifier : public DfsHloVisitor { operand_shapes, map->to_apply()->ComputeProgramShape(), map_dims)); } - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override { + Status HandleReduceWindow(HloInstruction* reduce_window) override { return CheckShape( reduce_window, ShapeInference::InferReduceWindowShape( @@ -296,11 +270,11 @@ class ShapeVerifier : public DfsHloVisitor { pad->padding_config())); } - Status HandleSend(HloInstruction* send) override { + Status HandleSend(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleRecv(HloInstruction* recv) override { + Status HandleRecv(HloInstruction*) override { return tensorflow::Status::OK(); } @@ -335,7 +309,7 @@ class ShapeVerifier : public DfsHloVisitor { batch_norm_grad->feature_index())); } - Status FinishVisit(HloInstruction* root) override { + Status FinishVisit(HloInstruction*) override { return tensorflow::Status::OK(); } diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc index 40df0dc355..9987ab4aee 100644 --- a/tensorflow/compiler/xla/service/inliner.cc +++ b/tensorflow/compiler/xla/service/inliner.cc @@ -43,11 +43,7 @@ class InlinerVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override; + Status HandleMap(HloInstruction* map) override; // Runs the visitor on a computation. StatusOr Run(HloComputation* computation); @@ -67,10 +63,8 @@ StatusOr InlinerVisitor::Run(HloComputation* computation) { return changed_; } -Status InlinerVisitor::HandleMap( - HloInstruction* map, tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice /*static_operands*/) { +Status InlinerVisitor::HandleMap(HloInstruction* map) { + HloComputation* function = map->to_apply(); HloInstruction& root = *function->root_instruction(); // TODO(b/29249531): Add DCE pass to remove unused HloComputations. // Only inlining functions that are simply a single operation until a better @@ -91,7 +85,7 @@ Status InlinerVisitor::HandleMap( if (root.opcode() != HloOpcode::kConstant) { std::vector params; for (int64 o = 0; o < root.operands().size(); o++) { - params.push_back(operands[root.operand(o)->parameter_number()]); + params.push_back(map->operands()[root.operand(o)->parameter_number()]); } HloInstruction* placed_instruction = computation_->AddInstruction( root.CloneWithNewOperands(map->shape(), params)); diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc index a2af2580ff..bc683a1880 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc @@ -72,8 +72,8 @@ Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) { return Status::OK(); } -Status FusedIrEmitter::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status FusedIrEmitter::HandleConstant(HloInstruction* constant) { + const Literal& literal = constant->literal(); llvm::Constant* initializer = llvm_ir::ConvertLiteralToIrConstant(literal, module_); llvm::GlobalVariable* global = new llvm::GlobalVariable( @@ -88,9 +88,10 @@ Status FusedIrEmitter::HandleConstant(HloInstruction* constant, return Status::OK(); } -Status FusedIrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status FusedIrEmitter::HandleGetTupleElement( + HloInstruction* get_tuple_element) { // Lookup ir value for 'operand'. + auto operand = get_tuple_element->operand(0); auto it = gte_values_.find(operand); if (it == gte_values_.end()) { return Unimplemented( @@ -128,9 +129,8 @@ Status FusedIrEmitter::HandleParameter(HloInstruction* parameter) { return Status::OK(); } -Status FusedIrEmitter::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status FusedIrEmitter::HandleTuple(HloInstruction* tuple) { + tensorflow::gtl::ArraySlice operands(tuple->operands()); std::vector operand_elemental_ir_types; for (HloInstruction* operand : operands) { operand_elemental_ir_types.push_back(llvm_ir::PrimitiveTypeToIrType( diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h index a44da51378..9ad7cd82cb 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h +++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h @@ -47,18 +47,14 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault { Status DefaultAction(HloInstruction* hlo) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleParameter(HloInstruction* parameter) override; // Emits the ir value for each element in the tuple. - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; + Status HandleTuple(HloInstruction* tuple) override; Status FinishVisit(HloInstruction* root) override; diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc index bf3bb2ddf0..b92017c6cb 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc @@ -86,8 +86,7 @@ Status LogicalBufferAnalysis::DefaultAction(HloInstruction* hlo_instruction) { return Status::OK(); } -Status LogicalBufferAnalysis::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { +Status LogicalBufferAnalysis::HandleGetTupleElement(HloInstruction*) { // GetTupleElement does not create buffers. return Status::OK(); } @@ -99,24 +98,19 @@ Status LogicalBufferAnalysis::HandleCopy(HloInstruction* copy) { return Status::OK(); } -Status LogicalBufferAnalysis::HandleBitcast(HloInstruction* bitcast) { +Status LogicalBufferAnalysis::HandleBitcast(HloInstruction*) { // A kBitcast instruction aliases its operand. That is, the buffer of its // result *is* the buffer of its operand. return Status::OK(); } -Status LogicalBufferAnalysis::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status LogicalBufferAnalysis::HandleTuple(HloInstruction* tuple) { // A Tuple instruction only creates the top-level buffer. NewLogicalBuffer(tuple, /*index=*/{}); return Status::OK(); } -Status LogicalBufferAnalysis::HandleSelect(HloInstruction* select, - HloInstruction* /*pred*/, - HloInstruction* on_true, - HloInstruction* on_false) { +Status LogicalBufferAnalysis::HandleSelect(HloInstruction* select) { // Select allocates a new buffer and then shallow copies the on_true or // on_false buffer into this new buffer. NewLogicalBuffer(select, /*index=*/{}); diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.h b/tensorflow/compiler/xla/service/logical_buffer_analysis.h index de9fe1b0a4..a82e83ec5c 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.h +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.h @@ -56,16 +56,11 @@ class LogicalBufferAnalysis : public DfsHloVisitorWithDefault { void NewLogicalBuffer(HloInstruction* instruction, const ShapeIndex& index); Status DefaultAction(HloInstruction* hlo_instruction) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleSelect(HloInstruction* select) override; // A map from the buffer ID to the logical buffer std::vector> logical_buffers_; diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc index f7dee93aad..df537bd7c1 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc @@ -200,13 +200,14 @@ Status TuplePointsToAnalysis::DefaultAction(HloInstruction* hlo_instruction) { } Status TuplePointsToAnalysis::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { + HloInstruction* get_tuple_element) { // GetTupleElement forwards a pointer to a particular element of the tuple // operand. int64 element_index = get_tuple_element->tuple_index(); PointsToSet& points_to_set = CreateEmptyPointsToSet(get_tuple_element); - const PointsToSet& operand_points_to_set = *PerInst(operand)->points_to_set; + const PointsToSet& operand_points_to_set = + *PerInst(get_tuple_element->operand(0))->points_to_set; // Copy the points-to set (and tuple sources) at index {element_index} of the // operand to the points-to set for this GetTupleElement instruction. @@ -252,9 +253,8 @@ Status TuplePointsToAnalysis::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } -Status TuplePointsToAnalysis::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status TuplePointsToAnalysis::HandleTuple(HloInstruction* tuple) { + tensorflow::gtl::ArraySlice operands(tuple->operands()); PointsToSet& points_to_set = CreateEmptyPointsToSet(tuple); points_to_set.AddPointedToBuffer( logical_buffer_analysis_->GetBuffer(tuple, /*index=*/{}), @@ -292,10 +292,7 @@ Status TuplePointsToAnalysis::HandleTuple( return Status::OK(); } -Status TuplePointsToAnalysis::HandleSelect(HloInstruction* select, - HloInstruction* /*pred*/, - HloInstruction* on_true, - HloInstruction* on_false) { +Status TuplePointsToAnalysis::HandleSelect(HloInstruction* select) { // Select allocates a new buffer and then shallow copies the on_true or // on_false buffer into this new buffer. Which side is chosen cannot be // determined statically so conservatively set the points-to set to the union @@ -303,6 +300,8 @@ Status TuplePointsToAnalysis::HandleSelect(HloInstruction* select, // // First create a copy of the on_true points-to set (and tuple sources), then // add in elements of the on_false points-to set (tuple sources). + auto on_true = select->operand(1); + auto on_false = select->operand(2); PointsToSet& points_to_set = CreateCopiedPointsToSet(select, on_true); const PointsToSet& false_points_to_set = *PerInst(on_false)->points_to_set; points_to_set.ForEachMutableElement( diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h index 30dabb56bd..e6157a1ed1 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h @@ -247,16 +247,11 @@ class TuplePointsToAnalysis : public DfsHloVisitorWithDefault { Status VerifyBuffer(const LogicalBuffer& buffer) const; Status DefaultAction(HloInstruction* hlo_instruction) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleSelect(HloInstruction* select) override; string ToString() const; -- GitLab From 6d1263cdf8ee8323513f984553dbeb070865fd0c Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Mon, 30 Oct 2017 19:57:47 -0700 Subject: [PATCH 553/573] [XLA] Remove dead opcode kIndex. PiperOrigin-RevId: 173987428 --- tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 1 - tensorflow/compiler/xla/service/hlo_instruction.cc | 3 --- tensorflow/compiler/xla/service/hlo_matchers.h | 1 - tensorflow/compiler/xla/service/hlo_opcode.cc | 3 --- tensorflow/compiler/xla/service/hlo_opcode.h | 1 - tensorflow/compiler/xla/service/inliner.cc | 1 - .../compiler/xla/service/instruction_fusion.cc | 1 - tensorflow/compiler/xla/service/shape_inference.cc | 13 ------------- tensorflow/compiler/xla/service/user_computation.cc | 2 -- tensorflow/compiler/xla/tools/parser/hlo_parser.cc | 1 - tensorflow/compiler/xla/xla_data.proto | 8 -------- 11 files changed, 35 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index 11edf49130..d7bdd4117d 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -849,7 +849,6 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kGe: case HloOpcode::kGt: case HloOpcode::kImag: - case HloOpcode::kIndex: case HloOpcode::kIsFinite: case HloOpcode::kLe: case HloOpcode::kLog: diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 1de4c4a115..b1bfd3e674 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -1164,7 +1164,6 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( break; case HloOpcode::kRecv: case HloOpcode::kSend: - case HloOpcode::kIndex: case HloOpcode::kTrace: LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_); } @@ -1551,7 +1550,6 @@ bool HloInstruction::IdenticalSlowPath( return dimensions() == other.dimensions(); // These opcodes are not yet supported. - case HloOpcode::kIndex: case HloOpcode::kInfeed: case HloOpcode::kOutfeed: case HloOpcode::kSort: @@ -2277,7 +2275,6 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { return visitor->HandleRecv(this); // These opcodes are not handled here. - case HloOpcode::kIndex: case HloOpcode::kTrace: break; } diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h index 5440ed2eda..bc5ed029a4 100644 --- a/tensorflow/compiler/xla/service/hlo_matchers.h +++ b/tensorflow/compiler/xla/service/hlo_matchers.h @@ -74,7 +74,6 @@ HLO_MATCHER(Fusion); HLO_MATCHER(Ge); HLO_MATCHER(GetTupleElement); HLO_MATCHER(Gt); -HLO_MATCHER(Index); HLO_MATCHER(Infeed); HLO_MATCHER(IsFinite); HLO_MATCHER(Le); diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc index d94c4da5ea..157d19f5a9 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.cc +++ b/tensorflow/compiler/xla/service/hlo_opcode.cc @@ -95,8 +95,6 @@ string HloOpcodeString(HloOpcode opcode) { return "greater-than"; case HloOpcode::kImag: return "imag"; - case HloOpcode::kIndex: - return "index"; case HloOpcode::kInfeed: return "infeed"; case HloOpcode::kIsFinite: @@ -218,7 +216,6 @@ StatusOr StringToHloOpcode(const string& opcode_name) { {"greater-than-or-equal-to", HloOpcode::kGe}, {"get-tuple-element", HloOpcode::kGetTupleElement}, {"greater-than", HloOpcode::kGt}, - {"index", HloOpcode::kIndex}, {"infeed", HloOpcode::kInfeed}, {"is-finite", HloOpcode::kIsFinite}, {"less-than-or-equal-to", HloOpcode::kLe}, diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h index 8090e4c82e..07c2d26f00 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.h +++ b/tensorflow/compiler/xla/service/hlo_opcode.h @@ -61,7 +61,6 @@ enum class HloOpcode { kGetTupleElement, kGt, kImag, - kIndex, kInfeed, kIsFinite, kLe, diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc index 9987ab4aee..5c193fceb9 100644 --- a/tensorflow/compiler/xla/service/inliner.cc +++ b/tensorflow/compiler/xla/service/inliner.cc @@ -71,7 +71,6 @@ Status InlinerVisitor::HandleMap(HloInstruction* map) { // profitability model for inlining is defined. if (hlo_query::AllOperandsAreParameters(root)) { if (root.opcode() == HloOpcode::kFusion || - root.opcode() == HloOpcode::kIndex || root.opcode() == HloOpcode::kParameter || root.opcode() == HloOpcode::kTrace) { // Cloning not supported for these instructions. diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc index fae3ca8ad2..0d1b7bc109 100644 --- a/tensorflow/compiler/xla/service/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/instruction_fusion.cc @@ -99,7 +99,6 @@ namespace xla { case HloOpcode::kDot: case HloOpcode::kExp: case HloOpcode::kFusion: - case HloOpcode::kIndex: case HloOpcode::kLog: case HloOpcode::kMap: case HloOpcode::kParameter: diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index 0458932a73..791d17365b 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -97,8 +97,6 @@ BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) { return BINOP_ADD; case HloOpcode::kSubtract: return BINOP_SUB; - case HloOpcode::kIndex: - return BINOP_INDEX; case HloOpcode::kDivide: return BINOP_DIV; case HloOpcode::kEq: @@ -830,17 +828,6 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( broadcast_dimensions)); return ShapeUtil::ChangeElementType(shape, PRED); } - case BINOP_INDEX: - if (ShapeUtil::Rank(lhs) > 0 && ShapeUtil::Rank(rhs) == 0) { - tensorflow::gtl::ArraySlice dimensions = - AsInt64Slice(lhs.dimensions()); - dimensions.pop_front(); - return ShapeUtil::MakeShape(lhs.element_type(), dimensions); - } - return Unimplemented("cannot infer shape for operation: %s <%s> %s", - ShapeUtil::HumanString(lhs).c_str(), - BinaryOperation_Name(operation).c_str(), - ShapeUtil::HumanString(rhs).c_str()); default: return Unimplemented( "not yet implemented; infer binary op shape: %s; lhs: %s; rhs: %s", diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index 0bdeffaf25..006c814996 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -96,8 +96,6 @@ HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) { return HloOpcode::kAdd; case BINOP_SUB: return HloOpcode::kSubtract; - case BINOP_INDEX: - return HloOpcode::kIndex; case BINOP_DIV: return HloOpcode::kDivide; case BINOP_EQ: diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc index 7c1eaa9f7f..5dd8ec6636 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -405,7 +405,6 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, case HloOpcode::kInfeed: case HloOpcode::kOutfeed: case HloOpcode::kBatchNormGrad: - case HloOpcode::kIndex: case HloOpcode::kTrace: return TokenError(StrCat("parsing not yet implemented for op: ", HloOpcodeString(opcode))); diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 2a8dc682a1..080e3c4267 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -700,14 +700,6 @@ enum BinaryOperation { // Dot product, matrix multiply. BINOP_DOT = 12; - // Indexes into the LHS with the RHS. - // - // If the RHS is higher-rank, this is a gather operation. - // - // Note: currently out of bounds indices may crash the underlying XLA - // machine. - BINOP_INDEX = 13; - // Element-wise maximum. BINOP_MAX = 14; -- GitLab From 113be57466d36ab7086794475cf4579f3e6b940b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 20:08:55 -0700 Subject: [PATCH 554/573] A few profiler improvements 1. Track the full allocation history of each tensor, visualized in timeline. 2. Better ProfileContext for tracing step selection. 3. Small bug fix. PiperOrigin-RevId: 173988293 --- .../internal/advisor/tfprof_advisor_test.cc | 2 +- .../profiler/internal/print_model_analysis.cc | 6 +- .../profiler/internal/print_model_analysis.h | 4 +- .../core/profiler/internal/tfprof_graph.cc | 2 +- .../core/profiler/internal/tfprof_node.cc | 16 +- .../core/profiler/internal/tfprof_node.h | 135 +++++++++++--- .../core/profiler/internal/tfprof_scope.cc | 2 +- .../core/profiler/internal/tfprof_stats.cc | 33 ++-- .../core/profiler/internal/tfprof_stats.h | 10 +- .../core/profiler/internal/tfprof_timeline.cc | 169 ++++++++++-------- .../core/profiler/internal/tfprof_timeline.h | 30 ++-- .../profiler/internal/tfprof_timeline_test.cc | 2 +- tensorflow/core/profiler/profiler.cc | 1 + tensorflow/core/profiler/tfprof_log.proto | 7 + tensorflow/python/profiler/model_analyzer.py | 7 +- .../python/profiler/model_analyzer_test.py | 2 +- tensorflow/python/profiler/profile_context.py | 139 ++++++++++---- .../python/profiler/profile_context_test.py | 44 +++++ 18 files changed, 429 insertions(+), 182 deletions(-) diff --git a/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc b/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc index c39d44b7fa..d05143aff9 100644 --- a/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc +++ b/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc @@ -48,7 +48,7 @@ class TFProfAdvisorTest : public ::testing::Test { for (const auto& attr : attrs) { (*def->mutable_attr())[attr.first].set_s(attr.second); } - std::unique_ptr node(new TFGraphNode(def, -1)); + std::unique_ptr node(new TFGraphNode(def, -1, nullptr)); NodeExecStats node_stat; node_stat.set_all_start_micros(start_miros); diff --git a/tensorflow/core/profiler/internal/print_model_analysis.cc b/tensorflow/core/profiler/internal/print_model_analysis.cc index 575ae182ee..7a0d590262 100644 --- a/tensorflow/core/profiler/internal/print_model_analysis.cc +++ b/tensorflow/core/profiler/internal/print_model_analysis.cc @@ -119,8 +119,8 @@ void DeleteProfiler() { } } -void AddStep(int64 step, const string* graph, const string* run_meta, - const string* op_log) { +double AddStep(int64 step, const string* graph, const string* run_meta, + const string* op_log) { CHECK(tf_stat); CHECK(graph && !graph->empty()); @@ -144,6 +144,7 @@ void AddStep(int64 step, const string* graph, const string* run_meta, op_log_ptr->ParseFromString(*op_log); tf_stat->AddOpLogProto(std::move(op_log_ptr)); } + return tf_stat->run_coverage(); } string Profile(const string* command, const string* options) { @@ -154,6 +155,7 @@ string Profile(const string* command, const string* options) { } void WriteProfile(const string* filename) { + CHECK(tf_stat); CHECK(filename) << "empty file name when asking to write profile."; tf_stat->WriteProfile(*filename); } diff --git a/tensorflow/core/profiler/internal/print_model_analysis.h b/tensorflow/core/profiler/internal/print_model_analysis.h index e4d01041a8..31ff5b07b0 100644 --- a/tensorflow/core/profiler/internal/print_model_analysis.h +++ b/tensorflow/core/profiler/internal/print_model_analysis.h @@ -35,8 +35,8 @@ bool NewProfiler(const string* graph, const string* op_log); void DeleteProfiler(); -void AddStep(int64 step, const string* graph, const string* run_meta, - const string* op_log); +double AddStep(int64 step, const string* graph, const string* run_meta, + const string* op_log); // Write the profiler's profile to a proto buffer. void WriteProfile(const string* filename); diff --git a/tensorflow/core/profiler/internal/tfprof_graph.cc b/tensorflow/core/profiler/internal/tfprof_graph.cc index 3766365bf8..db7ae3b397 100644 --- a/tensorflow/core/profiler/internal/tfprof_graph.cc +++ b/tensorflow/core/profiler/internal/tfprof_graph.cc @@ -31,7 +31,7 @@ GraphNode* TFGraph::CreateParentNode(const string& name) { node_defs_.back()->set_name(name); node_defs_.back()->set_op(kTFGraphParent); parent_nodes_[name] = std::unique_ptr( - new TFGraphNode(node_defs_.back().get(), -1)); + new TFGraphNode(node_defs_.back().get(), -1, nullptr)); nodes_map_[name] = std::unique_ptr(new GraphNode(parent_nodes_[name].get())); return nodes_map_[name].get(); diff --git a/tensorflow/core/profiler/internal/tfprof_node.cc b/tensorflow/core/profiler/internal/tfprof_node.cc index f283fafc0f..671b65d708 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.cc +++ b/tensorflow/core/profiler/internal/tfprof_node.cc @@ -19,19 +19,15 @@ limitations under the License. namespace tensorflow { namespace tfprof { -namespace { bool CountAsAcceleratorTime(const string& device) { return device.find("stream:all") != device.npos; } - bool CountAsCPUTime(const string& device) { return RE2::FullMatch(device, ".*/(device:gpu|gpu|device:cpu|cpu|device:sycl):\\d+"); } - bool IsCanonicalDevice(const string& device) { return CountAsCPUTime(device); } -} // namespace // Notes about start and end time from the NodeExecStats proto: // For GPU, there is no difference between op_end_rel_micros and // all_end_rel_micros. All are kernel times. @@ -89,16 +85,28 @@ void ExecStep::AddMemoryStats(const string& dev, } exec_.set_memory_intialized(true); + int accelerator_allocator_cnt = 0; for (const auto& mem : step_stat.memory()) { // TODO(xpan): Fix this hack. Currently the allocator name seems quite // ad-hoc. if (mem.allocator_name().find("GPU") == mem.allocator_name().npos) { continue; } + ++accelerator_allocator_cnt; exec_.set_allocator_bytes_in_use( std::max(static_cast(exec_.allocator_bytes_in_use()), static_cast(mem.allocator_bytes_in_use()))); + Allocation allocation; + for (const auto& alloc : mem.allocation_records()) { + allocation.add_allocation_records()->MergeFrom(alloc); + } + allocations_.push_back(allocation); } + if (accelerator_allocator_cnt > 1) { + fprintf(stderr, "found %d gpu allocator for 1 node\n", + accelerator_allocator_cnt); + } + int64 total_output_bytes = 0; for (const auto& output : step_stat.output()) { if (output.has_tensor_description() && diff --git a/tensorflow/core/profiler/internal/tfprof_node.h b/tensorflow/core/profiler/internal/tfprof_node.h index 34bc0a581d..e2d0563a07 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.h +++ b/tensorflow/core/profiler/internal/tfprof_node.h @@ -105,8 +105,22 @@ class ExecStep { const { return op_execs_; } + const std::map>>& cpu_execs() + const { + return cpu_execs_; + } + int64 all_start_micros() const { return exec_.all_start_micros(); } int64 latest_end_micros() const { return exec_.latest_end_micros(); } + int64 lastest_schedule_end_micros() const { + int64 ret = 0; + for (const auto& exec : cpu_execs_) { + for (const auto& pair : exec.second) { + ret = std::max(ret, pair.first + pair.second); + } + } + return ret; + } int64 requested_bytes() const { return exec_.requested_bytes(); } int64 peak_bytes() const { return exec_.peak_bytes(); } @@ -127,6 +141,8 @@ class ExecStep { return exec_.allocator_bytes_in_use(); } + const std::vector& allocations() const { return allocations_; } + const ExecProfile& ToProto() { exec_.mutable_accelerator_execs()->clear(); for (const auto& e : accelerator_execs_) { @@ -161,6 +177,11 @@ class ExecStep { mem_pb.set_ptr(mem.second.second); } + exec_.mutable_allocations()->Clear(); + for (const auto& r : allocations_) { + exec_.add_allocations()->MergeFrom(r); + } + return exec_; } @@ -175,6 +196,8 @@ class ExecStep { cpu_execs_.clear(); op_execs_.clear(); + allocations_.clear(); + for (const auto& exec_time : exec_.accelerator_execs()) { auto& exec = accelerator_execs_[exec_time.first]; auto& op_exec = op_execs_[exec_time.first]; @@ -196,6 +219,10 @@ class ExecStep { mem.first = output_mem.second.bytes(); mem.second = output_mem.second.ptr(); } + + for (const auto& r : exec_.allocations()) { + allocations_.push_back(r); + } } private: @@ -215,6 +242,9 @@ class ExecStep { std::set devices_; // output_idx -> {output_bytes, memory_ptr} std::map> output_memory_; + + // The history of accelerator allocations and deallocations of this step. + std::vector allocations_; }; #define GRAPH_NODE_BYTES(type) \ @@ -238,11 +268,15 @@ class ExecStep { class TFGraphNode { public: TFGraphNode(const ProfileNode& node, const ProfileProto& profile, - const std::map* id_to_string) { + const std::map* id_to_string, + const std::map>* nodes_map) { + nodes_map_ = nodes_map; FromProto(node, profile, id_to_string); } - TFGraphNode(const NodeDef* node, int64 id) { + TFGraphNode(const NodeDef* node, int64 id, + const std::map>* nodes_map) { + nodes_map_ = nodes_map; node_.set_id(id); node_.set_name(node->name()); node_.set_op(node->op()); @@ -269,17 +303,9 @@ class TFGraphNode { op_types_.insert(node->op()); } - void AddInput(TFGraphNode* input, int32 output_idx, int input_idx) { - src_output_idx_[input->name()] = output_idx; - - inputs_[input_idx] = input->name(); - const auto& output_shape = input->output_shapes().find(output_idx); - // Always create an empty vec even if the shape info might be missing. - std::vector& shape_vec = input_shapes_[input_idx]; - if (output_shape != input->output_shapes().end()) { - shape_vec.assign(output_shape->second.begin(), - output_shape->second.end()); - } + void AddInput(const string& input, int64 output_index, int input_idx) { + inputs_[input_idx] = input; + src_output_idx_[input] = output_index; } void AddOpType(const string& op_type) { op_types_.insert(op_type); } @@ -416,9 +442,6 @@ class TFGraphNode { } const std::map& inputs() const { return inputs_; } - const std::map& src_output_idx() const { - return src_output_idx_; - } // Number of times the graph node is executed. When step < 0, the // average number of times executed across all steps. @@ -526,14 +549,30 @@ class TFGraphNode { return exec->second.latest_end_micros(); } + int64 lastest_schedule_end_micros(int64 step) const { + auto exec = execs_.find(step); + if (exec == execs_.end()) { + return 0; + } + return exec->second.lastest_schedule_end_micros(); + } + const std::map>>& op_execs( int64 step) const { auto exec = execs_.find(step); if (exec == execs_.end()) { - return empty_op_execs_; + return empty_execs_; } return exec->second.op_execs(); } + const std::map>>& cpu_execs( + int64 step) const { + auto exec = execs_.find(step); + if (exec == execs_.end()) { + return empty_execs_; + } + return exec->second.cpu_execs(); + } const std::map& all_op_execs() const { return execs_; } @@ -551,12 +590,12 @@ class TFGraphNode { } return exec->second.host_temp_bytes(); } - int64 accelerator_persistent_bytes(int64 step) const { - auto exec = execs_.find(step); - if (exec == execs_.end()) { - return 0; + int64 accelerator_persistent_bytes() const { + int64 persistent_bytes = 0; + for (const auto& exec : execs_) { + persistent_bytes += exec.second.accelerator_persistent_bytes(); } - return exec->second.accelerator_persistent_bytes(); + return persistent_bytes; } int64 host_persistent_bytes(int64 step) const { auto exec = execs_.find(step); @@ -581,6 +620,14 @@ class TFGraphNode { return exec->second.allocator_bytes_in_use(); } + const std::vector& allocations(int64 step) const { + auto exec = execs_.find(step); + if (exec == execs_.end()) { + return empty_allocations_; + } + return exec->second.allocations(); + } + int64 parameters() const { if (!shape().empty()) { int64 params = 1; @@ -628,18 +675,44 @@ class TFGraphNode { const std::map>& output_shapes() const { return output_shapes_; } - const std::map>& input_shapes() const { - return input_shapes_; + + const std::map> input_shapes() const { + std::map> input_shapes; + for (const auto& inp : inputs_) { + // Always create an empty vec even if the shape info might be missing. + std::vector& shape_vec = input_shapes[inp.first]; + if (!nodes_map_) continue; + auto input_it = nodes_map_->find(inp.second); + if (input_it == nodes_map_->end()) continue; + auto output_it = src_output_idx_.find(inp.second); + if (output_it == src_output_idx_.end()) continue; + + const TFGraphNode* input_node = input_it->second.get(); + if (!input_node) continue; + const auto& output_shapes = input_node->output_shapes(); + const auto& output_shape = output_shapes.find(output_it->second); + if (output_shape == output_shapes.end()) continue; + + if (output_shape != input_node->output_shapes().end()) { + shape_vec.assign(output_shape->second.begin(), + output_shape->second.end()); + } + } + return input_shapes; } private: + // maps graph node name to TFGraphNode. Not owned. + const std::map>* nodes_map_; + // inputs to the node. input index -> input node name. std::map inputs_; + // The output index of the source node. std::map src_output_idx_; - + // proto for serialize/deserialized representation of the node. ProfileNode node_; - + // Python call stack that creates the name. std::unique_ptr call_stack_; - + // Shape of the node (e.g. Variable) if available. std::vector shape_; // Won't missing input_idx. But some shapes might be empty (unknown). std::map> input_shapes_; @@ -651,8 +724,10 @@ class TFGraphNode { std::map execs_; + // Placeholder for empty cases. std::map> empty_output_memory_; - std::map>> empty_op_execs_; + std::map>> empty_execs_; + std::vector empty_allocations_; }; class TFMultiGraphNode { @@ -806,6 +881,10 @@ class TFMultiGraphNode { }; bool IsPlacedOnAccelerator(const string& device); +bool CountAsAcceleratorTime(const string& device); +bool CountAsCPUTime(const string& device); +bool IsCanonicalDevice(const string& device); + } // namespace tfprof } // namespace tensorflow diff --git a/tensorflow/core/profiler/internal/tfprof_scope.cc b/tensorflow/core/profiler/internal/tfprof_scope.cc index 128b296d5c..988bed71cc 100644 --- a/tensorflow/core/profiler/internal/tfprof_scope.cc +++ b/tensorflow/core/profiler/internal/tfprof_scope.cc @@ -35,7 +35,7 @@ ScopeNode* TFScope::CreateParentNode(const string& name) { node_defs_.back()->set_name(name); node_defs_.back()->set_op(kTFScopeParent); parent_nodes_[name] = std::unique_ptr( - new TFGraphNode(node_defs_.back().get(), -1)); + new TFGraphNode(node_defs_.back().get(), -1, nullptr)); nodes_map_[name] = std::unique_ptr(new ScopeNode(parent_nodes_[name].get())); return nodes_map_[name].get(); diff --git a/tensorflow/core/profiler/internal/tfprof_stats.cc b/tensorflow/core/profiler/internal/tfprof_stats.cc index b4b98141f3..7943c075e0 100644 --- a/tensorflow/core/profiler/internal/tfprof_stats.cc +++ b/tensorflow/core/profiler/internal/tfprof_stats.cc @@ -36,7 +36,9 @@ bool CreateRunMetadataNode(const string& name, NodeDef* def) { } def->set_name(name); // TODO(xpan): Better operation type. - def->set_op("RunTimeOp"); + // This is because some times a node doesn't have a op type, + // so we use node name as the op type. + def->set_op(name); return true; } } // namespace @@ -86,7 +88,7 @@ TFStats::TFStats(const string& filename, } for (const auto& node_pb : profile.nodes()) { std::unique_ptr node( - new TFGraphNode(node_pb.second, profile, &id_to_string_)); + new TFGraphNode(node_pb.second, profile, &id_to_string_, &nodes_map_)); nodes_map_.insert(std::pair>( node_pb.second.name(), std::move(node))); } @@ -178,12 +180,14 @@ const MultiGraphNodeProto& TFStats::ShowMultiGraphNode( void TFStats::AddGraph(std::unique_ptr graph) { std::map node_defs; + bool node_added = false; for (const NodeDef& node : graph->node()) { if (nodes_map_.find(node.name()) != nodes_map_.end()) { continue; } - nodes_map_[node.name()] = - std::unique_ptr(new TFGraphNode(&node, nodes_map_.size())); + node_added = true; + nodes_map_[node.name()] = std::unique_ptr( + new TFGraphNode(&node, nodes_map_.size(), &nodes_map_)); node_defs[node.name()] = &node; } for (auto it = node_defs.begin(); it != node_defs.end(); it++) { @@ -192,6 +196,7 @@ void TFStats::AddGraph(std::unique_ptr graph) { string node_input = it->second->input(i); int output_idx = 0; // input name format can be: "^node:src_output" + // if not :src_output, then it's the first one (further verify?) auto prefix_pos = node_input.find(":"); if (prefix_pos != node_input.npos) { std::vector input_parts = str_util::Split(node_input, ":"); @@ -204,15 +209,18 @@ void TFStats::AddGraph(std::unique_ptr graph) { if (node_input.substr(0, 1) == "^") { node_input = node_input.substr(1); } - auto input_node = nodes_map_.find(node_input); - // TODO(xpan): P1: Add the input even if it doesn't exist yet, because - // this can be a partial graph. - if (input_node == nodes_map_.end()) { - continue; - } - node->AddInput(input_node->second.get(), output_idx, i); + // Delay input TFGraphNode retrieval as late as possible. + // In long run, when we have TensorFlow runtime graph, the + // graph connection should be dynamic and per-step. + node->AddInput(node_input, output_idx, i); } } + if (node_added) { + graph_view_.reset(nullptr); + scope_view_.reset(nullptr); + op_view_.reset(nullptr); + code_view_.reset(nullptr); + } } void TFStats::AddOpLogProto(std::unique_ptr op_log) { @@ -263,10 +271,11 @@ void TFStats::AddRunMeta(int64 step, std::unique_ptr run_meta) { NodeDef def; if (CreateRunMetadataNode(name, &def)) { nodes_map_[name] = std::unique_ptr( - new TFGraphNode(&def, nodes_map_.size())); + new TFGraphNode(&def, nodes_map_.size(), &nodes_map_)); nodes_map_.at(name)->AddStepStat(step, dev_stat.device(), node_stat); } } else { + covered_nodes_.insert(node->second->id()); node->second->AddStepStat(step, dev_stat.device(), node_stat); } } diff --git a/tensorflow/core/profiler/internal/tfprof_stats.h b/tensorflow/core/profiler/internal/tfprof_stats.h index bb4baea738..d46d923556 100644 --- a/tensorflow/core/profiler/internal/tfprof_stats.h +++ b/tensorflow/core/profiler/internal/tfprof_stats.h @@ -66,6 +66,9 @@ class TFStats { } const std::set& steps() const { return steps_; } bool has_code_traces() const { return has_code_traces_; } + double run_coverage() const { + return covered_nodes_.size() / (nodes_map_.size() + 1e-10); + } void BuildView(const string& cmd); void BuildAllViews(); @@ -104,13 +107,16 @@ class TFStats { std::unique_ptr code_view_; std::unique_ptr op_view_; std::unique_ptr ckpt_reader_; - // Store TFGraphNode instead of TFGraphNode* to avoid large number of - // dynamic alloc. + // TODO(xpan): Store TFGraphNode instead of TFGraphNode* to avoid large + // number of dynamic alloc. + // Maps from graph node name to TFGraphNode. std::map> nodes_map_; GraphNodeProto empty_graph_node_; MultiGraphNodeProto empty_multi_graph_node_; std::map id_to_string_; + // Graph nodes covered by RunMetdata, that is traced with run time stats. + std::set covered_nodes_; }; } // namespace tfprof diff --git a/tensorflow/core/profiler/internal/tfprof_timeline.cc b/tensorflow/core/profiler/internal/tfprof_timeline.cc index 1732574cc4..bdb000747d 100644 --- a/tensorflow/core/profiler/internal/tfprof_timeline.cc +++ b/tensorflow/core/profiler/internal/tfprof_timeline.cc @@ -25,6 +25,8 @@ limitations under the License. namespace tensorflow { namespace tfprof { namespace { +int kMaxDisplayedMemNode = 10; + string GetTimeDevName(const string& dev) { if (dev.find("stream") != dev.npos) { return strings::StrCat("Op execution threads: ", dev); @@ -85,14 +87,41 @@ void ChromeTraceFormatter::EmitFlowEnd(const string& name, int64 ts, int64 pid, events_.push_back(event); } -void ChromeTraceFormatter::EmitCounter(const string& category, - const string& name, int64 pid, int64 ts, - const string& device, int64 bytes) { - Json::Value event = CreateEvent("C", category, name, pid, 0, ts); +void ChromeTraceFormatter::EmitCounter( + const string& category, const string& name, int64 pid, int64 ts, + const string& device, int64 bytes, + const std::map>& tensor_mem) { + Json::Value event = CreateEvent("C", category, "Allocated Bytes", pid, 0, ts); Json::Value args(Json::objectValue); - args[device] = Json::Value(bytes); + args["Allocator Bytes in Use"] = Json::Value(bytes); event["args"] = args; events_.push_back(event); + + // TODO(xpan): chrome://tracing is not ideal visualization for memory. + // It would be great to have a customized UI for it. + Json::Value event2 = + CreateEvent("C", category, "Top Allocations", pid + 1, 0, ts); + Json::Value args2(Json::objectValue); + // Need to reserve the same args for all locations. + for (int i = 1; i < kMaxDisplayedMemNode; ++i) { + args2[strings::Printf("Top Allocation %02d", i)] = Json::Value("N/A"); + } + int count = 0; + for (auto it = tensor_mem.rbegin(); it != tensor_mem.rend(); ++it) { + for (const string& t : it->second) { + if (bytes < it->first || count >= kMaxDisplayedMemNode) { + break; + } + args2[strings::Printf("Top Allocation %02d", count)] = + Json::Value(strings::StrCat(it->first / 1000000.0, " MB from ", t)); + ++count; + bytes -= it->first; + } + } + args2[strings::StrCat("Not Displayed")] = + Json::Value(strings::Printf("%.2f MB", bytes / 1000000.0)); + event2["args"] = args2; + events_.push_back(event2); } string ChromeTraceFormatter::Format() { @@ -119,71 +148,28 @@ void MemoryTracker::TrackNode(int64 step, const GraphNode* node) { if (!node->Trackable(step)) { return; } + Device& dev = devices_[node->node->canonical_device()]; - int64 end_micros = node->node->latest_end_micros(step); - if (node->node->accelerator_persistent_bytes(step) != 0) { - string tensor_name = strings::StrCat(node->name(), ":", -1); - dev.earliest_ref[tensor_name] = node->node->all_start_micros(step); - dev.tensor_size[tensor_name] = - node->node->accelerator_persistent_bytes(step); - // TODO(xpan): Need latest_ref? - } - if (node->node->accelerator_temp_bytes(step)) { - string tensor_name = strings::StrCat(node->name(), ":", -2); - dev.earliest_ref[tensor_name] = node->node->all_start_micros(step); - dev.latest_ref[tensor_name] = end_micros; - dev.tensor_size[tensor_name] = node->node->accelerator_temp_bytes(step); - } - if (node->node->allocator_bytes_in_use(step) > 0) { - dev.allocator_stats[end_micros] = node->node->allocator_bytes_in_use(step); - } -} -void MemoryTracker::TrackNodeConnection(int64 step, const GraphNode* node, - const GraphNode* src) { - if (!node->Trackable(step) || !src->Trackable(step)) { - return; - } - const auto& output_idx = node->node->src_output_idx().find(src->name()); - if (output_idx == node->node->src_output_idx().end()) { - return; - } - const auto& output = src->node->output_memory(step).find(output_idx->second); - if (output == src->node->output_memory(step).end()) { - return; + std::map allocs; + for (const auto& alloc : node->node->allocations(step)) { + for (const auto& r : alloc.allocation_records()) { + allocs[r.alloc_micros()] += r.alloc_bytes(); + dev.tracked_allocations[r.alloc_micros()] += r.alloc_bytes(); + } } - int64 output_bytes = output->second.first; - uint64 output_ptr = output->second.second; - - Device& src_dev = devices_[src->node->canonical_device()]; - string tensor_name = strings::StrCat(output_ptr); - if (output_ptr == 0) { - fprintf(stderr, "output no ptr\n"); - tensor_name = strings::StrCat(src->node->name(), ":", output_idx->second); + dev.tracked_allocations[0] += node->node->accelerator_persistent_bytes(); + allocs[0] += node->node->accelerator_persistent_bytes(); + + int64 last = 0; + std::map& aggregate_allocs = dev.tensor_allocs[node->name()]; + for (auto it = allocs.begin(); it != allocs.end(); ++it) { + last += it->second; + aggregate_allocs[it->first] = last; } - - src_dev.tensor_size[tensor_name] = output_bytes; - src_dev.earliest_ref[tensor_name] = src->node->all_start_micros(step); - - int64 src_end_micros = src->node->latest_end_micros(step); - - if (src->node->canonical_device() != node->node->canonical_device()) { - int64 transfer_micros = - (src_end_micros + node->node->all_start_micros(step)) / 2; - src_dev.latest_ref[tensor_name] = - std::max(src_dev.latest_ref[tensor_name], transfer_micros); - - Device& dest_dev = devices_[node->node->canonical_device()]; - string dest_tensor_name = - strings::StrCat(tensor_name, node->node->canonical_device()); - dest_dev.tensor_size[dest_tensor_name] = output_bytes; - dest_dev.earliest_ref[dest_tensor_name] = transfer_micros; - dest_dev.latest_ref[dest_tensor_name] = - std::max(dest_dev.latest_ref[dest_tensor_name], - node->node->latest_end_micros(step)); - } else { - src_dev.latest_ref[tensor_name] = std::max( - src_dev.latest_ref[tensor_name], node->node->latest_end_micros(step)); + int64 end_micros = node->node->lastest_schedule_end_micros(step); + if (end_micros > 0 && node->node->allocator_bytes_in_use(step) > 0) { + dev.allocations[end_micros] = node->node->allocator_bytes_in_use(step); } } @@ -222,22 +208,24 @@ void Timeline::GenerateGraphTimeline(const std::vector& gnodes) { for (GraphNode* gnode : gnodes) { AllocateTimeNodes(gnode); } + // To save memory, we only track cross-device (canonical device) flows. for (auto& process : tnodes_) { + if (!IsCanonicalDevice(process.first)) continue; for (auto& tn : process.second) { TimeNode* tnode = tn.second.get(); for (GraphNode* inp : tnode->node->children) { if (!inp->account || !inp->Trackable(step_)) { continue; } - TrackNodeConnection(tnode->node, inp); - for (const auto& kernel_execs : inp->node->op_execs(step_)) { - if (process.first == kernel_execs.first) { - // Not interested in flow withthin the same device. + for (const auto& execs : inp->node->cpu_execs(step_)) { + if (!IsCanonicalDevice(execs.first)) continue; + if (process.first == execs.first) { + // Not interested in flow within the same device. continue; } - for (const auto& exec : kernel_execs.second) { + for (const auto& exec : execs.second) { int64 start_micros = exec.first; - auto cprocess = tnodes_.find(kernel_execs.first); + auto cprocess = tnodes_.find(execs.first); if (cprocess == tnodes_.end()) continue; auto ctn = cprocess->second.find(start_micros); if (ctn == cprocess->second.end()) continue; @@ -258,7 +246,6 @@ void Timeline::GenerateGraphTimeline(const std::vector& gnodes) { Json::Value args(Json::objectValue); args["name"] = Json::Value(tnode->name()); - args["op"] = Json::Value(tnode->name()); chrome_formatter_.EmitRegion(node.first, tnode->exec_micros, process.first, lane.first, "Op", tnode->name(), args); @@ -280,12 +267,40 @@ void Timeline::GenerateGraphTimeline(const std::vector& gnodes) { for (const auto& dev : mem_tracker_.devices()) { int64 pid = AllocatePID(); chrome_formatter_.EmitPID(GetMemoryLaneName(dev.first), pid); + int64 pid2 = AllocatePID(); + chrome_formatter_.EmitPID(GetMemoryLaneName(dev.first) + " allocations", + pid2); + const MemoryTracker::Device& device = dev.second; - for (const auto& alloc_stats : device.allocator_stats) { - chrome_formatter_.EmitCounter("Memory", "Memory Series", pid, - alloc_stats.first, dev.first, - alloc_stats.second); + int64 max_bytes_in_use = 0; + int64 cur_bytes_in_use = 0; + int64 last_point = 0; + for (const auto& alloc : device.allocations) { + cur_bytes_in_use = alloc.second; + max_bytes_in_use = std::max(max_bytes_in_use, cur_bytes_in_use); + // Do not plot too dense to reduce file size. + int64 ts = alloc.first; + if (ts - last_point < 100) continue; + last_point = ts; + + std::map> tensor_mem; + for (const auto& tensor_alloc_it : dev.second.tensor_allocs) { + const auto& tensor_alloc = tensor_alloc_it.second; + auto it = tensor_alloc.lower_bound(ts); + if (it != tensor_alloc.begin()) { + --it; + } + if (it->second > 0) { + tensor_mem[it->second].push_back(tensor_alloc_it.first); + } + } + chrome_formatter_.EmitCounter("Memory", "Memory Series", pid, ts, + dev.first, cur_bytes_in_use, tensor_mem); + } + if (IsPlacedOnAccelerator(dev.first)) { + fprintf(stdout, "%s peak memory: %.2f MB\n", dev.first.c_str(), + max_bytes_in_use / 1000000.0); } } OutputTimeline(); diff --git a/tensorflow/core/profiler/internal/tfprof_timeline.h b/tensorflow/core/profiler/internal/tfprof_timeline.h index 6c62d1046f..b8174cdecb 100644 --- a/tensorflow/core/profiler/internal/tfprof_timeline.h +++ b/tensorflow/core/profiler/internal/tfprof_timeline.h @@ -28,10 +28,12 @@ namespace tfprof { typedef std::map Event; +// Class for generating timeline json output. class ChromeTraceFormatter { public: ChromeTraceFormatter() {} - + // The following methods creates timeline nodes. See chrome tracing format + // document for details. Json::Value CreateEvent(const string& ph, const string& category, const string& name, int64 pid, int64 tid, int64 ts); @@ -47,22 +49,27 @@ class ChromeTraceFormatter { int64 flow_id); void EmitCounter(const string& category, const string& name, int64 pid, - int64 ts, const string& device, int64 bytes); + int64 ts, const string& device, int64 bytes, + const std::map>& tensor_mem); string Format(); private: + // A event is a visualization unit in timeline. std::vector events_; std::vector metadata_; }; +// A process (time series of events) in the timeline. class Process { public: Process(const string& device, int64 pid) : device(device), pid(pid) {} // Each lane is a map from start_time to end_time. std::vector> lanes; + // device for the time series. string device; + // unique id for the time series. int64 pid; }; @@ -96,19 +103,16 @@ class MemoryTracker { public: class Device { public: - // The first 3 fields are predicted. - std::map tensor_size; - std::map earliest_ref; - std::map latest_ref; + // map from tensor name to a pair of . + std::map> tensor_allocs; // ground truth memory stats. time->bytes. - std::map allocator_stats; + std::map allocations; + // tracked allocations, might miss some bytes. + std::map tracked_allocations; }; void TrackNode(int64 step, const GraphNode* node); - void TrackNodeConnection(int64 step, const GraphNode* node, - const GraphNode* src); - const std::map& devices() const { return devices_; } private: @@ -130,13 +134,9 @@ class Timeline { void GenerateCodeTimeline(const CodeNode* node); + private: void TrackNode(const GraphNode* node) { mem_tracker_.TrackNode(step_, node); } - void TrackNodeConnection(GraphNode* node, GraphNode* src) { - mem_tracker_.TrackNodeConnection(step_, node, src); - } - - private: void OutputTimeline(); template diff --git a/tensorflow/core/profiler/internal/tfprof_timeline_test.cc b/tensorflow/core/profiler/internal/tfprof_timeline_test.cc index babae395ba..91eac0cf76 100644 --- a/tensorflow/core/profiler/internal/tfprof_timeline_test.cc +++ b/tensorflow/core/profiler/internal/tfprof_timeline_test.cc @@ -71,7 +71,7 @@ TEST_F(TFProfTimelineTest, GraphView) { string dump_str; TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file + "_0", &dump_str)); - EXPECT_EQ(1754536562981488144ull, Hash64(dump_str)); + EXPECT_EQ(7932146665024565912ull, Hash64(dump_str)); } TEST_F(TFProfTimelineTest, ScopeView) { diff --git a/tensorflow/core/profiler/profiler.cc b/tensorflow/core/profiler/profiler.cc index 96e0b06bf3..a5e513aa21 100644 --- a/tensorflow/core/profiler/profiler.cc +++ b/tensorflow/core/profiler/profiler.cc @@ -234,6 +234,7 @@ int Run(int argc, char** argv) { return 1; } tf_stat->AddRunMeta(i, std::move(run_meta)); + fprintf(stdout, "run graph coverage: %.2f\n", tf_stat->run_coverage()); } } diff --git a/tensorflow/core/profiler/tfprof_log.proto b/tensorflow/core/profiler/tfprof_log.proto index a1410c7c79..f92301133a 100644 --- a/tensorflow/core/profiler/tfprof_log.proto +++ b/tensorflow/core/profiler/tfprof_log.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package tensorflow.tfprof; import "tensorflow/core/framework/attr_value.proto"; +import "tensorflow/core/framework/step_stats.proto"; // It specifies the Python callstack that creates an op. message CodeDef { @@ -89,6 +90,10 @@ message ProfileNode { map execs = 12; } +message Allocation { + repeated AllocationRecord allocation_records = 1; +} + message ExecProfile { // Can be larger than 1 if run multiple times in loop. int64 run_count = 1; @@ -107,6 +112,8 @@ message ExecProfile { map output_memory = 17; + repeated Allocation allocations = 18; + repeated string devices = 6; // Total bytes requested by the op. diff --git a/tensorflow/python/profiler/model_analyzer.py b/tensorflow/python/profiler/model_analyzer.py index 2071325c7b..040a489163 100644 --- a/tensorflow/python/profiler/model_analyzer.py +++ b/tensorflow/python/profiler/model_analyzer.py @@ -157,6 +157,7 @@ class Profiler(object): op_log: optional. tensorflow::tfprof::OpLogProto proto. Used to define extra op types. """ + self._coverage = 0.0 self._graph = graph # pylint: disable=protected-access op_log = tfprof_logger._merge_default_with_oplog( @@ -183,7 +184,7 @@ class Profiler(object): self._graph, run_meta=run_meta) # pylint: enable=protected-access # TODO(xpan): P1: Better to find the current graph. - print_mdl.AddStep( + self._coverage = print_mdl.AddStep( step, self._graph.as_graph_def(add_shapes=True).SerializeToString(), run_meta.SerializeToString(), op_log.SerializeToString()) @@ -274,6 +275,10 @@ class Profiler(object): print_mdl.Profile('advise'.encode('utf-8'), opts.SerializeToString())) return advise_pb + def _write_profile(self, filename): + """Writes the profile to a file.""" + print_mdl.WriteProfile(filename) + def profile(graph, run_meta=None, diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py index 2578fc3e87..17c87bea92 100644 --- a/tensorflow/python/profiler/model_analyzer_test.py +++ b/tensorflow/python/profiler/model_analyzer_test.py @@ -159,7 +159,7 @@ class PrintModelAnalysisTest(test.TestCase): with gfile.Open(outfile, 'r') as f: # pylint: disable=line-too-long self.assertEqual( - 'node name | # parameters | # float_ops | assigned devices | op types | op count (run|defined) | input shapes\n_TFProfRoot (--/451 params, --/11.34k flops, _kTFScopeParent, --/8|--/36, )\n Conv2D (0/0 params, 5.83k/5.83k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x6x6x3|1:3x3x3x6)\n Conv2D_1 (0/0 params, 4.61k/4.61k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x3x3x6|1:2x2x6x12)\n DW (3x3x3x6, 162/162 params, 0/324 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:3x3x3x6|1:3x3x3x6)\n DW/Initializer (0/0 params, 0/324 flops, _kTFScopeParent, 0/0|1/7, )\n DW/Initializer/random_normal (0/0 params, 162/324 flops, Add, 0/0|1/6, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/mul (0/0 params, 162/162 flops, Mul, 0/0|1/1, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:3x3x3x6)\n DW2 (2x2x6x12, 288/288 params, 0/576 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW2/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:2x2x6x12|1:2x2x6x12)\n DW2/Initializer (0/0 params, 0/576 flops, _kTFScopeParent, 0/0|1/7, )\n DW2/Initializer/random_normal (0/0 params, 288/576 flops, Add, 0/0|1/6, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW2/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/mul (0/0 params, 288/288 flops, Mul, 0/0|1/1, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:2x2x6x12)\n ScalarW (1, 1/1 params, 0/2 flops, VariableV2|_trainable_variables, 0/0|1/10, )\n ScalarW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer (0/0 params, 0/2 flops, _kTFScopeParent, 0/0|1/7, )\n ScalarW/Initializer/random_normal (0/0 params, 1/2 flops, Add, 0/0|1/6, 0:1|1:1)\n ScalarW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:0)\n ScalarW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/mul (0/0 params, 1/1 flops, Mul, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/read (0/0 params, 0/0 flops, Identity, 0/0|1/1, 0:1)\n _retval_Conv2D_1_0_0 (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|RunTimeOp, 1/1|1/1, )\n init (0/0 params, 0/0 flops, NoOp, 0/0|1/1, 0:1|1:3x3x3x6|2:2x2x6x12)\n zeros (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Const, 1/1|1/1, )\n', + 'node name | # parameters | # float_ops | assigned devices | op types | op count (run|defined) | input shapes\n_TFProfRoot (--/451 params, --/11.34k flops, _kTFScopeParent, --/8|--/36, )\n Conv2D (0/0 params, 5.83k/5.83k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x6x6x3|1:3x3x3x6)\n Conv2D_1 (0/0 params, 4.61k/4.61k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x3x3x6|1:2x2x6x12)\n DW (3x3x3x6, 162/162 params, 0/324 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:3x3x3x6|1:3x3x3x6)\n DW/Initializer (0/0 params, 0/324 flops, _kTFScopeParent, 0/0|1/7, )\n DW/Initializer/random_normal (0/0 params, 162/324 flops, Add, 0/0|1/6, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/mul (0/0 params, 162/162 flops, Mul, 0/0|1/1, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:3x3x3x6)\n DW2 (2x2x6x12, 288/288 params, 0/576 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW2/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:2x2x6x12|1:2x2x6x12)\n DW2/Initializer (0/0 params, 0/576 flops, _kTFScopeParent, 0/0|1/7, )\n DW2/Initializer/random_normal (0/0 params, 288/576 flops, Add, 0/0|1/6, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW2/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/mul (0/0 params, 288/288 flops, Mul, 0/0|1/1, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:2x2x6x12)\n ScalarW (1, 1/1 params, 0/2 flops, VariableV2|_trainable_variables, 0/0|1/10, )\n ScalarW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer (0/0 params, 0/2 flops, _kTFScopeParent, 0/0|1/7, )\n ScalarW/Initializer/random_normal (0/0 params, 1/2 flops, Add, 0/0|1/6, 0:1|1:1)\n ScalarW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:0)\n ScalarW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/mul (0/0 params, 1/1 flops, Mul, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/read (0/0 params, 0/0 flops, Identity, 0/0|1/1, 0:1)\n _retval_Conv2D_1_0_0 (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|_retval_Conv2D_1_0_0, 1/1|1/1, )\n init (0/0 params, 0/0 flops, NoOp, 0/0|1/1, 0:1|1:3x3x3x6|2:2x2x6x12)\n zeros (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Const, 1/1|1/1, )\n', f.read()) # pylint: enable=line-too-long diff --git a/tensorflow/python/profiler/profile_context.py b/tensorflow/python/profiler/profile_context.py index 0c31cf8f13..c7c7ad6301 100644 --- a/tensorflow/python/profiler/profile_context.py +++ b/tensorflow/python/profiler/profile_context.py @@ -20,6 +20,8 @@ from __future__ import print_function import contextlib import os +import random +import sys import threading from tensorflow.core.protobuf import config_pb2 @@ -31,6 +33,7 @@ from tensorflow.python.platform import gfile from tensorflow.python.profiler import model_analyzer from tensorflow.python.util import compat +WARMUP_STEPS = 10 MAX_TRACED_STEPS = 100 @@ -51,7 +54,9 @@ def _profiled_run(self, # Fast path if no need for profiling. if not self.profile_context._is_fast_path(): # Maybe trace this step. - if self.profile_context._should_trace(): + if self.profile_context._should_trace(self.graph, fetches): + if self.profile_context._debug: + sys.stderr.write('debug: tracing step: %d\n' % step) # Enable tracing, perform auto profiling or auto dump. if not run_metadata: run_metadata = config_pb2.RunMetadata() @@ -66,6 +71,8 @@ def _profiled_run(self, ret = self._profiler_run_internal( fetches, feed_dict, options, run_metadata) + if self.profile_context._debug: + self.profile_context._dump_file(run_metadata, 'run_meta_%d' % step) self.profile_context.profiler._graph = self.graph self.profile_context.profiler.add_step(step, run_metadata) @@ -80,6 +87,8 @@ def _profiled_run(self, to_profiles = self.profile_context._profile_candidates() for to_prof in to_profiles: cmd, opts, _ = to_prof + if self.profile_context._debug: + sys.stderr.write('debug: profiling %s step: %d\n' % (cmd, step)) if cmd == 'graph': self.profile_context.profiler.profile_graph(opts) elif cmd == 'scope': @@ -131,29 +140,43 @@ class ProfileContext(object): pre-defined steps. dump_steps: A list of steps to dump the profile to `profile_dir`. If None, use pre-defined steps. + enabled: If false, everything is disabled with minimal overhead. It allows + user to only enable profiling when needed. + debug: If true, also dumps the raw trace RunMetadata text file to + profile_dir. And print debugging message. Useful for bug report. """ def __init__(self, profile_dir, trace_steps=None, - dump_steps=None): + dump_steps=None, + enabled=True, + debug=False): + self._enabled = enabled + if not self._enabled: + return + + self._debug = debug if not profile_dir: raise ValueError('Must have a directory for profile.\n') self._profiler_dir = profile_dir if trace_steps is None: - self._trace_steps = set(list(range(10, 100, 3)) + - list(range(100, 10000, 1000))) + self._trace_steps = set() + self._auto_tracing = True else: if len(trace_steps) > MAX_TRACED_STEPS: raise ValueError('Only support tracing up to 100 steps.\n') self._trace_steps = set(trace_steps[:]) + self._auto_tracing = False if dump_steps is None: - self._dump_steps = set([100] + list(range(100, 10000, 2000))) + self._dump_steps = set([MAX_TRACED_STEPS]) else: self._dump_steps = set(dump_steps[:]) + self._rng = random.Random(111) + self._fetched = set() self._slow_path_steps = self._dump_steps | self._trace_steps self._trace_next_step = False self._dump_next_step = False @@ -173,6 +196,8 @@ class ProfileContext(object): will be run automatically at these integer steps. Each step is a session.run. """ + if not self._enabled: + return self._auto_profiles.append((cmd, options, profile_steps[:])) self._slow_path_steps |= set(profile_steps) self._trace_steps |= set(profile_steps) @@ -180,41 +205,82 @@ class ProfileContext(object): @property def profiler(self): """Returns the current profiler object.""" + if not self._enabled: + return None if not self._profiler: self._profiler = model_analyzer.Profiler(ops.get_default_graph()) return self._profiler def trace_next_step(self): - """Enables tracing and add traces to profiler at next step.""" + """Enables tracing and adds traces to profiler at next step.""" + if not self._enabled: + return self._trace_next_step = True + self._slow_path_steps.add(self._step) def dump_next_step(self): """Enable tracing and dump profiles at next step.""" + if not self._enabled: + return self._dump_next_step = True + self._slow_path_steps.add(self._step) def _is_fast_path(self): - if (self._step in self._slow_path_steps or - self._trace_next_step or - self._dump_next_step): + if self._step in self._slow_path_steps: + return False + # When user doesn't set the tracing steps explicitly, auto decide it. + if (self._auto_tracing and self._step > WARMUP_STEPS and + self._traced_steps <= MAX_TRACED_STEPS): return False return True - def _should_trace(self): + def _should_trace(self, graph, fetches): + """Whether should do tracing at current step.""" if self._traced_steps > MAX_TRACED_STEPS: return False - trace = self._step in self._trace_steps or self._trace_next_step - if trace: + # Check user-set tracing steps. + if self._step in self._trace_steps or self._trace_next_step: self._traced_steps += 1 - return trace + return True + + # If no user-set tracing steps set and passes warm up steps, auto trace. + if self._auto_tracing and self._step > WARMUP_STEPS: + # If the fetches have not been seen before, trace it. + with graph.as_default(): + fetch_names = [f.name for f in + session._FetchMapper.for_fetch(fetches).unique_fetches()] # pylint: disable=protected-access + fetch_name = '-'.join(sorted(fetch_names)) + if self._debug: + sys.stderr.write('debug: trace fetches: %s\n' % fetch_name) + if fetch_name not in self._fetched: + self._fetched.add(fetch_name) + self._traced_steps += 1 + return True + # If the trace coverage is low, does some random tracing. + if (self.profiler._coverage < 0.5 and self._step < MAX_TRACED_STEPS and # pylint: disable=protected-access + self._rng.randint(0, 10) < 2): + self._traced_steps += 1 + return True + return False def _maybe_dump(self): + """Maybe dump the profile file.""" if not (self._step in self._dump_steps or self._dump_next_step): return + if self._debug: + sys.stderr.write('debug: dumping file at step: %d\n' % self._step) if not gfile.Exists(self._profiler_dir): gfile.MakeDirs(self._profiler_dir) - print_mdl.WriteProfile( - os.path.join(compat.as_bytes(self._profiler_dir), - compat.as_bytes('profile_%d' % self._step))) + + filename = os.path.join(compat.as_bytes(self._profiler_dir), + compat.as_bytes('profile_%d' % self._step)) + self.profiler._write_profile(filename) # pylint: disable=protected-access + + def _dump_file(self, pb, basename): + if not gfile.Exists(self._profiler_dir): + gfile.MakeDirs(self._profiler_dir) + with gfile.Open(os.path.join(self._profiler_dir, basename), 'w') as f: + f.write('%s' % pb) @contextlib.contextmanager def _new_step(self): @@ -233,28 +299,33 @@ class ProfileContext(object): return to_profile def __enter__(self): - self.old_run = getattr(session.BaseSession, 'run', None) - self.old_init = getattr(session.BaseSession, '__init__', None) - if not self.old_run: - raise errors.InternalError(None, None, 'BaseSession misses run method.') - elif not self.old_init: - raise errors.InternalError(None, None, - 'BaseSession misses __init__ method.') - elif getattr(session.BaseSession, '_profiler_run_internal', None): - raise errors.InternalError(None, None, - 'Already in context or context not cleaned.') - elif getattr(session.BaseSession, '_profiler_init_internal', None): - raise errors.InternalError(None, None, - 'Already in context or context not cleaned.') + if self._enabled: + self.old_run = getattr(session.BaseSession, 'run', None) + self.old_init = getattr(session.BaseSession, '__init__', None) + if not self.old_run: + raise errors.InternalError(None, None, 'BaseSession misses run method.') + elif not self.old_init: + raise errors.InternalError(None, None, + 'BaseSession misses __init__ method.') + elif getattr(session.BaseSession, '_profiler_run_internal', None): + raise errors.InternalError(None, None, + 'Already in context or context not cleaned.') + elif getattr(session.BaseSession, '_profiler_init_internal', None): + raise errors.InternalError(None, None, + 'Already in context or context not cleaned.') + else: + setattr(session.BaseSession, 'run', _profiled_run) + setattr(session.BaseSession, '__init__', _profiled_init) + setattr(session.BaseSession, '_profiler_run_internal', self.old_run) + setattr(session.BaseSession, '_profiler_init_internal', self.old_init) + setattr(session.BaseSession, 'profile_context', self) + return self else: - setattr(session.BaseSession, 'run', _profiled_run) - setattr(session.BaseSession, '__init__', _profiled_init) - setattr(session.BaseSession, '_profiler_run_internal', self.old_run) - setattr(session.BaseSession, '_profiler_init_internal', self.old_init) - setattr(session.BaseSession, 'profile_context', self) return self def __exit__(self, exec_type, exec_value, exec_tb): + if not self._enabled: + return print_mdl.DeleteProfiler() setattr(session.BaseSession, 'run', self.old_run) setattr(session.BaseSession, '__init__', self.old_init) diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py index bbb49974ed..a623beee23 100644 --- a/tensorflow/python/profiler/profile_context_test.py +++ b/tensorflow/python/profiler/profile_context_test.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function import os + from tensorflow.python.client import session from tensorflow.python.framework import ops from tensorflow.python.ops import variables @@ -66,6 +67,49 @@ class ProfilerContextTest(test.TestCase): with gfile.Open(outfile, "r") as f: self.assertEqual(profile_str, f.read()) + def testAutoTracingInDeubMode(self): + ops.reset_default_graph() + x = lib.BuildFullModel() + + with profile_context.ProfileContext(test.get_temp_dir(), debug=True): + with session.Session() as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(10): + sess.run(x) + for f in gfile.ListDirectory(test.get_temp_dir()): + # Warm up, no tracing. + self.assertFalse("run_meta" in f) + sess.run(x) + self.assertTrue( + gfile.Exists(os.path.join(test.get_temp_dir(), "run_meta_11"))) + gfile.Remove(os.path.join(test.get_temp_dir(), "run_meta_11")) + # fetched already. + sess.run(x) + for f in gfile.ListDirectory(test.get_temp_dir()): + self.assertFalse("run_meta" in f) + + def testDisabled(self): + ops.reset_default_graph() + x = lib.BuildFullModel() + with profile_context.ProfileContext(test.get_temp_dir(), + enabled=False) as pctx: + with session.Session() as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(10): + sess.run(x) + self.assertTrue(pctx.profiler is None) + self.assertTrue( + getattr(session.BaseSession, "profile_context", None) is None) + + with profile_context.ProfileContext(test.get_temp_dir()) as pctx: + with session.Session() as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(10): + sess.run(x) + self.assertFalse(pctx.profiler is None) + self.assertFalse( + getattr(session.BaseSession, "profile_context", None) is None) + if __name__ == "__main__": test.main() -- GitLab From 17695212ccaffd214e1cc4f929afaa22dfb1d4c9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 20:49:51 -0700 Subject: [PATCH 555/573] [TF:XLA] Don't pass HLO operands in HandleAtan2. This makes it consistent with the rest of the Visit methods where we only pass the HLO itself. PiperOrigin-RevId: 173990595 --- tensorflow/compiler/xla/service/dfs_hlo_visitor.h | 6 +----- tensorflow/compiler/xla/service/hlo_instruction.cc | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h index e57a492dde..237cd8c31d 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h @@ -52,9 +52,6 @@ class HloInstruction; // "unimplemented" error status. // // Note: this may change to an iterator in the future for flexibility purposes. -// -// TODO(b/26548304): Stop passing in information about the visited -// instruction that is accessible from the instruction object itself. class DfsHloVisitor { public: DfsHloVisitor() {} @@ -110,8 +107,7 @@ class DfsHloVisitor { virtual Status HandleAbs(HloInstruction* abs) { return HandleElementwiseUnary(abs); } - virtual Status HandleAtan2(HloInstruction* atan2, HloInstruction* y, - HloInstruction* x) { + virtual Status HandleAtan2(HloInstruction* atan2) { return HandleElementwiseBinary(atan2); } virtual Status HandleRound(HloInstruction* round) { diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index b1bfd3e674..e6a4f68fb3 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2131,7 +2131,7 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kAbs: return visitor->HandleAbs(this); case HloOpcode::kAtan2: - return visitor->HandleAtan2(this, operands_[0], operands_[1]); + return visitor->HandleAtan2(this); case HloOpcode::kRoundNearestAfz: return visitor->HandleRound(this); case HloOpcode::kBatchNormTraining: -- GitLab From cd81bc8e09c7f551911276c5bfaafa6930f1961f Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Mon, 30 Oct 2017 20:51:27 -0700 Subject: [PATCH 556/573] Adds a PrefetchWithFn op to contrib/data. Alongwith the FunctionBufferingResource, this can be used to prefetch and fill up a buffer by making repeated function calls. Also fixes a TODO in the ProcessFLR implementation to respect alloc_attrs for Rendezvous calls. PiperOrigin-RevId: 173990680 --- tensorflow/BUILD | 1 + .../contrib/cmake/tf_core_kernels.cmake | 2 + tensorflow/contrib/cmake/tf_core_ops.cmake | 1 + tensorflow/contrib/cmake/tf_python.cmake | 2 + tensorflow/contrib/data/BUILD | 21 + tensorflow/contrib/data/kernels/BUILD | 29 ++ .../data/kernels/prefetching_kernels.cc | 378 ++++++++++++++++++ .../contrib/data/ops/prefetching_ops.cc | 58 +++ .../contrib/data/python/kernel_tests/BUILD | 15 + .../kernel_tests/prefetching_ops_test.py | 108 +++++ tensorflow/contrib/data/python/ops/BUILD | 45 +++ .../data/python/ops/prefetching_ops.py | 55 +++ tensorflow/core/common_runtime/function.cc | 48 ++- .../process_function_library_runtime.cc | 28 +- .../process_function_library_runtime.h | 21 +- .../core/common_runtime/rendezvous_util.cc | 56 ++- .../core/common_runtime/rendezvous_util.h | 30 +- .../common_runtime/rendezvous_util_test.cc | 11 +- .../core/distributed_runtime/graph_mgr.cc | 9 +- tensorflow/core/framework/function.h | 10 + tensorflow/core/kernels/function_ops.cc | 7 +- 21 files changed, 858 insertions(+), 77 deletions(-) create mode 100644 tensorflow/contrib/data/kernels/BUILD create mode 100644 tensorflow/contrib/data/kernels/prefetching_kernels.cc create mode 100644 tensorflow/contrib/data/ops/prefetching_ops.cc create mode 100644 tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py create mode 100644 tensorflow/contrib/data/python/ops/prefetching_ops.py diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 8667fd7c91..8e3aa1f97a 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -363,6 +363,7 @@ filegroup( "//tensorflow/contrib/crf:all_files", "//tensorflow/contrib/cudnn_rnn:all_files", "//tensorflow/contrib/data:all_files", + "//tensorflow/contrib/data/kernels:all_files", "//tensorflow/contrib/data/python/kernel_tests:all_files", "//tensorflow/contrib/data/python/ops:all_files", "//tensorflow/contrib/decision_trees/proto:all_files", diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake index 65565aad7e..f978c8ccd5 100644 --- a/tensorflow/contrib/cmake/tf_core_kernels.cmake +++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake @@ -69,6 +69,8 @@ if(tensorflow_BUILD_CONTRIB_KERNELS) "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc" + "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/prefetching_kernels.cc" + "${tensorflow_source_dir}/tensorflow/contrib/data/ops/prefetching_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc" diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake index 15e9a4c461..4a61ed7a35 100644 --- a/tensorflow/contrib/cmake/tf_core_ops.cmake +++ b/tensorflow/contrib/cmake/tf_core_ops.cmake @@ -81,6 +81,7 @@ GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_prediction "${tensorflow_source_dir}/t GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_quantiles "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_stats_accumulator "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(cudnn_rnn "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc") +GENERATE_CONTRIB_OP_LIBRARY(data_prefetching "${tensorflow_source_dir}/tensorflow/contrib/data/ops/prefetching_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(factorization_clustering "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/clustering_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(factorization_factorization "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/factorization_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(framework_variable "${tensorflow_source_dir}/tensorflow/contrib/framework/ops/variable_ops.cc") diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 1b9fd514fd..277818b159 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -769,6 +769,8 @@ GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_stats_accumulator_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_stats_accumulator_ops.py) GENERATE_PYTHON_OP_LIB("contrib_cudnn_rnn_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/cudnn_rnn/ops/gen_cudnn_rnn_ops.py) +GENERATE_PYTHON_OP_LIB("contrib_data_prefetching_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/data/python/ops/gen_prefetching_ops.py) GENERATE_PYTHON_OP_LIB("contrib_factorization_clustering_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/factorization/python/ops/gen_clustering_ops.py) GENERATE_PYTHON_OP_LIB("contrib_factorization_factorization_ops" diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD index b485d78f5c..eaede0e00e 100644 --- a/tensorflow/contrib/data/BUILD +++ b/tensorflow/contrib/data/BUILD @@ -4,6 +4,12 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load( + "//tensorflow:tensorflow.bzl", + "tf_custom_op_library", + "tf_gen_op_libs", +) + py_library( name = "data", srcs = ["__init__.py"], @@ -11,6 +17,7 @@ py_library( deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:iterator_ops", + "//tensorflow/contrib/data/python/ops:prefetching_py", "//tensorflow/contrib/data/python/ops:readers", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:util", @@ -18,6 +25,20 @@ py_library( ], ) +tf_custom_op_library( + name = "_prefetching_ops.so", + srcs = [ + "ops/prefetching_ops.cc", + ], + deps = [ + "//tensorflow/contrib/data/kernels:prefetching_kernels", + ], +) + +tf_gen_op_libs( + op_lib_names = ["prefetching_ops"], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/data/kernels/BUILD b/tensorflow/contrib/data/kernels/BUILD new file mode 100644 index 0000000000..4cb53741eb --- /dev/null +++ b/tensorflow/contrib/data/kernels/BUILD @@ -0,0 +1,29 @@ +# Description: +# Contains kernels for datasets and iterators. +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +cc_library( + name = "prefetching_kernels", + srcs = ["prefetching_kernels.cc"], + deps = [ + "//tensorflow/core:framework_headers_lib", + "//third_party/eigen3", + "@protobuf_archive//:protobuf_headers", + ], + alwayslink = 1, +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), +) diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc new file mode 100644 index 0000000000..c9a3537c70 --- /dev/null +++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc @@ -0,0 +1,378 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_op_kernel.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { + +struct BufferElement { + // The producer sets `status` if getting the input element fails. + Status status; + // The buffered data element. + std::vector value; +}; + +using FunctionBufferCallback = std::function; + +class FunctionBufferingResource : public ResourceBase { + public: + FunctionBufferingResource(FunctionLibraryRuntime* lib, + const NameAttrList& func, int64 buffer_size, + const string& source_device, + const string& target_device, + const std::vector& func_args, + int64 thread_pool_size) + : lib_(lib), + func_(func), + buffer_size_(buffer_size), + source_device_(source_device), + target_device_(target_device), + func_args_(func_args), + thread_pool_(new thread::ThreadPool(Env::Default(), ThreadOptions(), + "buffer_resource", thread_pool_size, + false /* low_latency_hint */)), + handle_(kInvalidHandle), + is_buffering_(false), + end_of_sequence_(false), + cancelled_(false) { + runner_ = [this](std::function c) { + thread_pool_->Schedule(std::move(c)); + }; + } + + ~FunctionBufferingResource() override { + Cancel(); + { + mutex_lock l(mu_); + while (is_buffering_) { + cond_var_.wait(l); + } + } + delete thread_pool_; + } + + string DebugString() override { + return strings::StrCat("FunctionBufferingResource. Size: ", buffer_size_, + "; target_device: ", target_device_); + } + + // Instantiates the function the first time it's called. After that it caches + // the handle. + Status Instantiate() LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + // Re-use existing handle if it's been set, effectively caching it. + if (handle_ != kInvalidHandle) { + return Status::OK(); + } + AttrValueMap attr_values = func_.attr(); + AttrValue v; + v.set_s(target_device_); + AddAttr("_target", v, &attr_values); + + return lib_->Instantiate(func_.name(), AttrSlice(&attr_values), &handle_); + } + + // Returns true if we've got to the end of the sequence and exhausted the + // buffer. + bool Finished() LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + return end_of_sequence_ && buffer_.empty(); + } + + // Cancels any buffering / prefetching going on. + void Cancel() LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + cancelled_ = true; + } + + // If the buffer has anything, runs `callback` on the first element in the + // buffer, else schedules the `callback` to be called. Requires `args` and + // `lib` in case more function calls need to be scheduled. + void MaybeGet(FunctionBufferCallback callback) LOCKS_EXCLUDED(mu_) { + bool start_buffering = false; + bool produced_output = false; + BufferElement buffer_element; + { + mutex_lock l(mu_); + if (!is_buffering_ && !end_of_sequence_) { + start_buffering = true; + } + if (!buffer_.empty()) { + produced_output = true; + std::swap(buffer_element, buffer_.front()); + buffer_.pop_front(); + } else { + produced_output = false; + requests_.push_back(std::move(callback)); + } + } + if (produced_output) { + callback(buffer_element); + } + if (start_buffering) { + FillBuffer(); + } + } + + private: + void FillBuffer() LOCKS_EXCLUDED(mu_) { + FunctionLibraryRuntime::Handle handle; + std::vector cancellation_callbacks; + std::vector cancellation_buffer_elements; + bool cancelled = false; + { + mutex_lock l(mu_); + handle = handle_; + if (cancelled_) { + cancelled = true; + // Run through and fulfill all pending requests, if possible. + while (!requests_.empty()) { + if (!buffer_.empty()) { + cancellation_buffer_elements.push_back(std::move(buffer_.front())); + buffer_.pop_front(); + cancellation_callbacks.push_back(std::move(requests_.front())); + requests_.pop_front(); + } else { + LOG(ERROR) << "Buffer ran out of elements and we couldn't satisfy: " + << requests_.size() << " requests"; + break; + } + } + is_buffering_ = false; + } else { + is_buffering_ = true; + } + } + if (cancelled) { + for (int i = 0; i < cancellation_callbacks.size(); ++i) { + cancellation_callbacks[i](cancellation_buffer_elements[i]); + } + // We only wait on cond_var_ in the destructor, so there would atmost be + // one waiter to notify. + cond_var_.notify_one(); + return; + } + FunctionLibraryRuntime::Options opts; + // Copied from CapturedFunction::generate_step_id(); + opts.step_id = -std::abs(static_cast(random::New64())); + opts.runner = &runner_; + opts.source_device = source_device_; + AllocatorAttributes arg_alloc_attr; + arg_alloc_attr.set_on_host(true); + opts.args_alloc_attrs.push_back(arg_alloc_attr); + if (opts.source_device != target_device_) { + opts.remote_execution = true; + } + opts.create_rendezvous = true; + auto* rets = new std::vector; + lib_->Run(opts, handle, func_args_, rets, + [this, rets](const Status& status) { + FunctionBufferCallback callback = nullptr; + BufferElement buffer_front; + bool restart_buffering = false; + { + mutex_lock l(mu_); + BufferElement buffer_element; + buffer_element.status = status; + if (!status.ok()) { + end_of_sequence_ = true; + is_buffering_ = false; + buffer_.push_back(std::move(buffer_element)); + return; + } + buffer_element.value.swap(*rets); + buffer_.push_back(std::move(buffer_element)); + if (!requests_.empty()) { + buffer_front = std::move(buffer_.front()); + buffer_.pop_front(); + callback = std::move(requests_.front()); + requests_.pop_front(); + } + if (buffer_.size() < buffer_size_) { + restart_buffering = true; + } else { + is_buffering_ = false; + } + } + if (callback != nullptr) { + callback(buffer_front); + } + if (restart_buffering) { + FillBuffer(); + } + }); + } + + mutex mu_; + FunctionLibraryRuntime* lib_; + NameAttrList func_; + const int64 buffer_size_; + const string source_device_; + const string target_device_; + const std::vector func_args_; + thread::ThreadPool* thread_pool_; + FunctionLibraryRuntime::Handle handle_ GUARDED_BY(mu_); + std::deque buffer_ GUARDED_BY(mu_); + std::deque requests_ GUARDED_BY(mu_); + std::function)> runner_ = nullptr; + bool is_buffering_ GUARDED_BY(mu_); + bool end_of_sequence_ GUARDED_BY(mu_); + bool cancelled_ GUARDED_BY(mu_); + condition_variable cond_var_; +}; + +class FunctionBufferResourceHandleOp : public OpKernel { + public: + explicit FunctionBufferResourceHandleOp(OpKernelConstruction* ctx) + : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("buffer_size", &buffer_size_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("container", &container_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("shared_name", &name_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("thread_pool_size", &thread_pool_size_)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor* string_arg; + OP_REQUIRES_OK(ctx, ctx->input("string_arg", &string_arg)); + std::vector func_args; + func_args.push_back(*string_arg); + + // Obtain and canonicalize target_device. + const Tensor* target_arg; + OP_REQUIRES_OK(ctx, ctx->input("target_device", &target_arg)); + const string& target_device = + DeviceNameUtils::CanonicalizeDeviceName(target_arg->scalar()()); + + FunctionLibraryRuntime* lib = ctx->function_library(); + OP_REQUIRES(ctx, lib != nullptr, + errors::Internal("No function library is provided.")); + + const string& source_device = ctx->device()->name(); + + ContainerInfo cinfo; + OP_REQUIRES_OK(ctx, cinfo.Init(ctx->resource_manager(), def())); + // Create the resource. + FunctionBufferingResource* buffer; + OP_REQUIRES_OK( + ctx, ctx->resource_manager()->LookupOrCreate( + cinfo.container(), cinfo.name(), &buffer, + [lib, &source_device, &target_device, func_args, + this](FunctionBufferingResource** ptr) { + *ptr = new FunctionBufferingResource( + lib, func_, buffer_size_, source_device, target_device, + func_args, thread_pool_size_); + return Status::OK(); + })); + OP_REQUIRES_OK(ctx, buffer->Instantiate()); + + OP_REQUIRES_OK(ctx, MakeResourceHandleToOutput( + ctx, 0, cinfo.container(), cinfo.name(), + MakeTypeIndex())); + } + + private: + NameAttrList func_; + int64 buffer_size_; + string container_; + string name_; + int64 thread_pool_size_; +}; + +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource") + .Device(DEVICE_CPU) + .HostMemory("resource") + .HostMemory("string_arg") + .HostMemory("target_device"), + FunctionBufferResourceHandleOp); +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource") + .Device(DEVICE_GPU) + .HostMemory("resource") + .HostMemory("string_arg") + .HostMemory("target_device"), + FunctionBufferResourceHandleOp); +#if TENSORFLOW_USE_SYCL +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource") + .Device(DEVICE_SYCL) + .HostMemory("resource") + .HostMemory("string_arg") + .HostMemory("target_device"), + FunctionBufferResourceHandleOp); +#endif // TENSORFLOW_USE_SYCL + +// Prefetches and fills up a buffer by calling a function that provides the +// elements to buffer. +class FunctionBufferingResourceGetNextOp : public AsyncOpKernel { + public: + explicit FunctionBufferingResourceGetNextOp(OpKernelConstruction* ctx) + : AsyncOpKernel(ctx) {} + + ~FunctionBufferingResourceGetNextOp() override {} + + void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override { + ResourceHandle handle; + OP_REQUIRES_OK_ASYNC( + ctx, HandleFromInput(ctx, "function_buffer_resource", &handle), done); + FunctionBufferingResource* buffer = nullptr; + OP_REQUIRES_OK_ASYNC( + ctx, LookupResource(ctx, handle, &buffer), + done); + core::ScopedUnref s(buffer); + + if (buffer->Finished()) { + ctx->SetStatus(errors::OutOfRange("end_of_sequence")); + done(); + return; + } + + FunctionBufferCallback callback = + [ctx, done](const BufferElement& buffer_element) { + Status s = buffer_element.status; + if (!s.ok()) { + ctx->SetStatus(s); + done(); + return; + } + for (size_t i = 0; i < buffer_element.value.size(); ++i) { + ctx->set_output(i, buffer_element.value[i]); + } + done(); + }; + buffer->MaybeGet(std::move(callback)); + } +}; + +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext") + .Device(DEVICE_CPU) + .HostMemory("function_buffer_resource"), + FunctionBufferingResourceGetNextOp); +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext") + .Device(DEVICE_GPU) + .HostMemory("function_buffer_resource"), + FunctionBufferingResourceGetNextOp); +#if TENSORFLOW_USE_SYCL +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext") + .Device(DEVICE_SYCL) + .HostMemory("function_buffer_resource"), + FunctionBufferingResourceGetNextOp); +#endif // TENSORFLOW_USE_SYCL + +} // namespace tensorflow diff --git a/tensorflow/contrib/data/ops/prefetching_ops.cc b/tensorflow/contrib/data/ops/prefetching_ops.cc new file mode 100644 index 0000000000..23cb62b6f0 --- /dev/null +++ b/tensorflow/contrib/data/ops/prefetching_ops.cc @@ -0,0 +1,58 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" + +namespace tensorflow { + +REGISTER_OP("FunctionBufferingResource") + .Input("string_arg: string") + .Input("target_device: string") + .Output("resource: resource") + .Attr("shared_name: string") + .Attr("container: string") + .Attr("f: func") + .Attr("buffer_size: int") + .Attr("thread_pool_size: int") + .SetShapeFn(shape_inference::UnknownShape) + .Doc(R"doc( +Creates a resource that fills up a buffer by making function calls. + +string_arg: String argument to the function call. +target_device: Target device to execute the function on. +resource: Handle to the resource created. +f: Function to be executed. +buffer_size: Size of the buffer. +thread_pool_size: Size of the threadpool doing the prefetching. +container: If non-empty, this resource is placed in the given container. + Otherwise, a default container is used. +shared_name: If non-empty, this resource will be shared under the given name + across multiple sessions. +)doc"); + +REGISTER_OP("FunctionBufferingResourceGetNext") + .Input("function_buffer_resource: resource") + .Attr("output_types: list(type)") + .Output("output: output_types") + .SetShapeFn(shape_inference::UnknownShape) + .Doc(R"doc( +Gets the next element from a FunctionBufferingResource. + +function_buffer_resource: The FunctionBufferingResource handle. +output: A list of return values. +output_types: The type list for the return values. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 22a027f178..424eb19852 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -416,6 +416,21 @@ py_test( ], ) +py_test( + name = "prefetching_ops_test", + size = "small", + srcs = ["prefetching_ops_test.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:prefetching_py", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py new file mode 100644 index 0000000000..539c6f2155 --- /dev/null +++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py @@ -0,0 +1,108 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for prefetching_ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import itertools +import threading + +from tensorflow.contrib.data.python.ops import prefetching_ops +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import iterator_ops +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +class StagingAreaOpsTest(test.TestCase): + + def setUp(self): + self._event = threading.Event() + + def _prefetch_fn_helper(self, buffer_name, device0, device1): + worker_config = config_pb2.ConfigProto() + worker_config.device_count["CPU"] = 2 + + def gen(): + for i in itertools.count(start=1, step=1): + yield [i + 0.0] + if i == 6: + self._event.set() + + with ops.device(device0): + dataset_3 = dataset_ops.Dataset.from_generator(gen, (dtypes.float32)) + iterator_3 = dataset_3.make_one_shot_iterator() + iterator_3_handle = iterator_3.string_handle() + + @function.Defun(dtypes.string) + def _remote_fn(h): + remote_iterator = iterator_ops.Iterator.from_string_handle( + h, dataset_3.output_types, dataset_3.output_shapes) + return remote_iterator.get_next() + + target = constant_op.constant(device0) + with ops.device(device1): + buffer_resource_handle = prefetching_ops.function_buffering_resource( + f=_remote_fn, + target_device=target, + string_arg=iterator_3_handle, + buffer_size=3, + thread_pool_size=2, + shared_name=buffer_name) + + with ops.device(device1): + prefetch_op = prefetching_ops.function_buffering_resource_get_next( + function_buffer_resource=buffer_resource_handle, + output_types=[dtypes.float32]) + + with self.test_session(config=worker_config) as sess: + elem = sess.run(prefetch_op) + self.assertEqual(elem, [1.0]) + elem = sess.run(prefetch_op) + self.assertEqual(elem, [2.0]) + elem = sess.run(prefetch_op) + self.assertEqual(elem, [3.0]) + elem = sess.run(prefetch_op) + self.assertEqual(elem, [4.0]) + self._event.wait() + elem = sess.run(prefetch_op) + self.assertEqual(elem, [5.0]) + + def testSameDeviceCPU(self): + self._prefetch_fn_helper("same_device_cpu", + "/job:localhost/replica:0/task:0/cpu:0", + "/job:localhost/replica:0/task:0/cpu:0") + + def testDifferentDeviceCPU(self): + self._prefetch_fn_helper("diff_device_cpu", + "/job:localhost/replica:0/task:0/cpu:0", + "/job:localhost/replica:0/task:0/cpu:1") + + def testDifferentDeviceCPUGPU(self): + if not test_util.is_gpu_available(): + self.skipTest("No GPU available") + + self._prefetch_fn_helper("cpu_gpu", "/job:localhost/replica:0/task:0/cpu:0", + "/job:localhost/replica:0/task:0/gpu:0") + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index e0730488a1..1b81cf5be9 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -4,6 +4,13 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load( + "//tensorflow:tensorflow.bzl", + "tf_gen_op_wrapper_py", + "tf_kernel_library", +) +load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") + py_library( name = "dataset_ops", srcs = [ @@ -83,6 +90,44 @@ py_library( ], ) +tf_gen_op_wrapper_py( + name = "prefetching_ops", + out = "gen_prefetching_ops.py", + deps = ["//tensorflow/contrib/data:prefetching_ops_op_lib"], +) + +tf_kernel_library( + name = "prefetching_ops_kernels", + deps = [ + "//tensorflow/contrib/data/kernels:prefetching_kernels", + "//tensorflow/core:framework", + ], + alwayslink = 1, +) + +tf_custom_op_py_library( + name = "prefetching_py", + srcs = ["prefetching_ops.py"], + dso = ["//tensorflow/contrib/data:_prefetching_ops.so"], + kernels = [ + ":prefetching_ops_kernels", + "//tensorflow/contrib/data:prefetching_ops_op_lib", + ], + srcs_version = "PY2AND3", + deps = [ + ":prefetching_ops", + "//tensorflow/contrib/util:util_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:errors", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform", + "//tensorflow/python:state_ops", + "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py new file mode 100644 index 0000000000..cfe8012b56 --- /dev/null +++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py @@ -0,0 +1,55 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Python wrapper for prefetching_ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.data.python.ops import gen_prefetching_ops +from tensorflow.contrib.util import loader +from tensorflow.python.platform import resource_loader + +_prefetching_ops = loader.load_op_library( + resource_loader.get_path_to_datafile("../../_prefetching_ops.so")) + + +# TODO(rohanj): Add a python class that constructs resource in the __init__ +# method and provides a get_next() that calls the prefetch op. +def function_buffering_resource(string_arg, + target_device, + shared_name, + f, + buffer_size, + thread_pool_size=1, + container="", + name=None): + return gen_prefetching_ops.function_buffering_resource( + string_arg=string_arg, + target_device=target_device, + shared_name=shared_name, + f=f, + buffer_size=buffer_size, + thread_pool_size=thread_pool_size, + container=container, + name=name) + + +def function_buffering_resource_get_next(function_buffer_resource, + output_types, + name=None): + return gen_prefetching_ops.function_buffering_resource_get_next( + function_buffer_resource=function_buffer_resource, + output_types=output_types, + name=name) diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc index d886a02305..10356fc789 100644 --- a/tensorflow/core/common_runtime/function.cc +++ b/tensorflow/core/common_runtime/function.cc @@ -569,10 +569,8 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, string target_device = parent_->GetDeviceName(handle); string source_device = opts.source_device; Rendezvous* rendezvous = opts.rendezvous; - // TODO(rohanj): Handle alloc_attrs in Rendezvous::Args. - Rendezvous::Args rendez_args; - Status s = - parent_->GetDeviceContext(target_device, &rendez_args.device_context); + DeviceContext* device_context; + Status s = parent_->GetDeviceContext(target_device, &device_context); if (!s.ok()) { delete frame; delete exec_args; @@ -596,12 +594,14 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, std::vector* remote_args = new std::vector; ProcessFunctionLibraryRuntime::ReceiveTensorsAsync( source_device, target_device, "arg_", src_incarnation, args.size(), - rendez_args, rendezvous, remote_args, + device_context, {}, rendezvous, remote_args, [frame, remote_args, item, source_device, target_device, - target_incarnation, rendezvous, rendez_args, rets, done, + target_incarnation, rendezvous, device_context, rets, done, exec_args](const Status& status) { Status s = status; - s = frame->SetArgs(*remote_args); + if (s.ok()) { + s = frame->SetArgs(*remote_args); + } if (!s.ok()) { delete frame; delete remote_args; @@ -611,7 +611,7 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, } item->exec->RunAsync( *exec_args, [item, frame, rets, done, source_device, target_device, - target_incarnation, rendezvous, rendez_args, + target_incarnation, rendezvous, device_context, remote_args, exec_args](const Status& status) { item->Unref(); Status s = status; @@ -627,7 +627,7 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, } s = ProcessFunctionLibraryRuntime::SendTensors( target_device, source_device, "ret_", target_incarnation, - *rets, rendez_args, rendezvous); + *rets, device_context, {}, rendezvous); delete remote_args; delete exec_args; done(s); @@ -643,8 +643,18 @@ void FunctionLibraryRuntimeImpl::Run(const Options& opts, Handle handle, done(errors::Cancelled("")); return; } + Options run_opts = opts; + if (opts.create_rendezvous) { + Rendezvous* rendezvous = new IntraProcessRendezvous(device_mgr_); + run_opts.rendezvous = rendezvous; + run_opts.create_rendezvous = false; + done = [done, rendezvous](const Status& status) { + rendezvous->Unref(); + done(status); + }; + } if (!parent_->IsInstantiatedOnDevice(device_name_, handle)) { - parent_->Run(opts, handle, args, rets, done); + parent_->Run(run_opts, handle, args, rets, done); return; } const FunctionBody* fbody = GetFunctionBody(handle); @@ -658,20 +668,20 @@ void FunctionLibraryRuntimeImpl::Run(const Options& opts, Handle handle, done(s); return; } - DCHECK(opts.runner != nullptr); + DCHECK(run_opts.runner != nullptr); Executor::Args* exec_args = new Executor::Args; // Inherit the step_id from the caller. - exec_args->step_id = opts.step_id; - exec_args->rendezvous = opts.rendezvous; - exec_args->stats_collector = opts.stats_collector; + exec_args->step_id = run_opts.step_id; + exec_args->rendezvous = run_opts.rendezvous; + exec_args->stats_collector = run_opts.stats_collector; exec_args->call_frame = frame; - exec_args->cancellation_manager = opts.cancellation_manager; - exec_args->step_container = opts.step_container; - exec_args->runner = *opts.runner; + exec_args->cancellation_manager = run_opts.cancellation_manager; + exec_args->step_container = run_opts.step_container; + exec_args->runner = *run_opts.runner; - if (opts.remote_execution) { - RunRemote(opts, handle, args, rets, exec_args, item, done); + if (run_opts.remote_execution) { + RunRemote(run_opts, handle, args, rets, exec_args, item, done); return; } diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index 68ff28e4d8..c4114ff873 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -95,7 +95,8 @@ string ProcessFunctionLibraryRuntime::ObtainFunctionTarget( Status ProcessFunctionLibraryRuntime::SendTensors( const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, - gtl::ArraySlice tensors_to_send, const Rendezvous::Args& args, + gtl::ArraySlice tensors_to_send, DeviceContext* device_context, + const std::vector& alloc_attrs, Rendezvous* rendezvous) { std::vector keys; for (int i = 0; i < tensors_to_send.size(); ++i) { @@ -104,8 +105,8 @@ Status ProcessFunctionLibraryRuntime::SendTensors( target_device, name, FrameAndIter(0, 0)); keys.push_back(key); } - TF_RETURN_IF_ERROR( - SendTensorsToRendezvous(rendezvous, args, keys, tensors_to_send)); + TF_RETURN_IF_ERROR(SendTensorsToRendezvous( + rendezvous, device_context, alloc_attrs, keys, tensors_to_send)); return Status::OK(); } @@ -113,7 +114,8 @@ Status ProcessFunctionLibraryRuntime::SendTensors( void ProcessFunctionLibraryRuntime::ReceiveTensorsAsync( const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, int64 num_tensors, - const Rendezvous::Args& args, Rendezvous* rendezvous, + DeviceContext* device_context, + const std::vector& alloc_attrs, Rendezvous* rendezvous, std::vector* received_tensors, const StatusCallback& done) { std::vector keys; for (int64 i = 0; i < num_tensors; ++i) { @@ -123,7 +125,7 @@ void ProcessFunctionLibraryRuntime::ReceiveTensorsAsync( keys.push_back(key); } RecvOutputsFromRendezvousAsync( - rendezvous, args, keys, received_tensors, + rendezvous, device_context, alloc_attrs, keys, received_tensors, [done](const Status& status) { done(status); }); } @@ -265,8 +267,8 @@ void ProcessFunctionLibraryRuntime::Run( if (flr != nullptr) { auto rendezvous = opts.rendezvous; string source_device = opts.source_device; - Rendezvous::Args rendez_args; - Status s = GetDeviceContext(source_device, &rendez_args.device_context); + DeviceContext* device_context; + Status s = GetDeviceContext(source_device, &device_context); if (!s.ok()) { done(s); return; @@ -281,15 +283,18 @@ void ProcessFunctionLibraryRuntime::Run( // Send the args over to the target device. s = SendTensors(source_device, target_device, "arg_", src_incarnation, args, - rendez_args, rendezvous); + device_context, opts.args_alloc_attrs, rendezvous); if (!s.ok()) { done(s); return; } + const std::vector& rets_alloc_attrs = + opts.rets_alloc_attrs; std::vector* remote_rets = new std::vector; flr->Run(opts, handle, args, remote_rets, [source_device, target_device, target_incarnation, rendezvous, - remote_rets, rets, done, rendez_args](const Status& status) { + device_context, rets_alloc_attrs, remote_rets, rets, + done](const Status& status) { if (!status.ok()) { delete remote_rets; done(status); @@ -299,8 +304,9 @@ void ProcessFunctionLibraryRuntime::Run( delete remote_rets; // Now receive the return values from the target. ReceiveTensorsAsync(target_device, source_device, "ret_", - target_incarnation, num_returns, rendez_args, - rendezvous, rets, done); + target_incarnation, num_returns, + device_context, rets_alloc_attrs, rendezvous, + rets, done); }); return; } diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.h b/tensorflow/core/common_runtime/process_function_library_runtime.h index 9f03de0f76..85717739d0 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.h +++ b/tensorflow/core/common_runtime/process_function_library_runtime.h @@ -60,26 +60,33 @@ class ProcessFunctionLibraryRuntime { // Sends `tensors_to_send` from `source_device` to `target_device` using // `rendezvous`. `key_prefix` is used as a prefix for the keys sent to the - // Rendezvous. Method takes references on each of the `tensors_to_send`. - // Method doesn't block. + // Rendezvous. `device_context` should be the DeviceContext of the device + // doing the sending. `alloc_attrs` should either be empty or be the size of + // `tensors_to_send` and indicates how the input tensors are allocated. Method + // takes references on each of the `tensors_to_send`. Method doesn't block. static Status SendTensors(const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, gtl::ArraySlice tensors_to_send, - const Rendezvous::Args& args, + DeviceContext* device_context, + const std::vector& alloc_attrs, Rendezvous* rendezvous); typedef std::function StatusCallback; // Receives `received_tensors` from `target_device` (originally sent from // `source_device`) using `rendezvous`. Uses `key_prefix` to construct the - // keys to be retrieved. Method doesn't block and calls `done` when - // `num_tensors` are fetched. + // keys to be retrieved. `device_context` should be for the device receiving + // the tensors. `alloc_attrs` indicates how to allocate the received + // tensors and should either be empty or `num_tensors` in size. Method doesn't + // block and calls `done` when `num_tensors` are fetched. static void ReceiveTensorsAsync( const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, int64 num_tensors, - const Rendezvous::Args& args, Rendezvous* rendezvous, - std::vector* received_tensors, const StatusCallback& done); + DeviceContext* device_context, + const std::vector& alloc_attrs, + Rendezvous* rendezvous, std::vector* received_tensors, + const StatusCallback& done); static const char kDefaultFLRDevice[]; // Returns the FunctionLibraryRuntime for the corresponding device_name. diff --git a/tensorflow/core/common_runtime/rendezvous_util.cc b/tensorflow/core/common_runtime/rendezvous_util.cc index a0d409e773..a1e31016c2 100644 --- a/tensorflow/core/common_runtime/rendezvous_util.cc +++ b/tensorflow/core/common_runtime/rendezvous_util.cc @@ -16,35 +16,55 @@ limitations under the License. namespace tensorflow { -Status SendTensorsToRendezvous(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - gtl::ArraySlice tensors_to_send) { +Status SendTensorsToRendezvous( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, gtl::ArraySlice tensors_to_send) { if (keys.size() != tensors_to_send.size()) { return errors::InvalidArgument( "keys and tensors_to_send are not the same size. keys.size() = ", keys.size(), "; tensors_to_send.size() = ", tensors_to_send.size()); } + if (!alloc_attrs.empty() && (keys.size() != alloc_attrs.size())) { + return errors::InvalidArgument( + "keys and alloc_attrs are not the same size. ", + "keys.size() = ", keys.size(), + "; alloc_attrs.size() = ", alloc_attrs.size()); + } + Rendezvous::ParsedKey parsed; for (int i = 0; i < keys.size(); ++i) { + Rendezvous::Args rendez_args; + rendez_args.device_context = device_context; + if (!alloc_attrs.empty()) { + rendez_args.alloc_attrs = alloc_attrs[i]; + } TF_RETURN_IF_ERROR(Rendezvous::ParseKey(keys[i], &parsed)); TF_RETURN_IF_ERROR( - rendezvous->Send(parsed, args, tensors_to_send[i], false)); + rendezvous->Send(parsed, rendez_args, tensors_to_send[i], false)); } return Status::OK(); } -void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - std::vector* received_tensors, - const StatusCallback& done) { +void RecvOutputsFromRendezvousAsync( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, std::vector* received_tensors, + const StatusCallback& done) { if (keys.empty()) { done(Status::OK()); return; } + if (!alloc_attrs.empty() && (keys.size() != alloc_attrs.size())) { + done(errors::InvalidArgument( + "keys and alloc_attrs are not the same size. ", "keys.size() = ", + keys.size(), "; alloc_attrs.size() = ", alloc_attrs.size())); + } + received_tensors->reserve(keys.size()); - std::vector> arguments; + std::vector< + std::tuple> + arguments; for (int i = 0; i < keys.size(); ++i) { Rendezvous::ParsedKey parsed; Status s = Rendezvous::ParseKey(keys[i], &parsed); @@ -53,8 +73,12 @@ void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, done(s); return; } - arguments.push_back( - std::make_tuple(keys[i], &((*received_tensors)[i]), parsed)); + AllocatorAttributes alloc_attr; + if (!alloc_attrs.empty()) { + alloc_attr = alloc_attrs[i]; + } + arguments.emplace_back(keys[i], &((*received_tensors)[i]), parsed, + alloc_attr); } typedef struct { @@ -68,8 +92,12 @@ void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, const string& key = std::get<0>(p); Tensor* val = std::get<1>(p); Rendezvous::ParsedKey parsed = std::get<2>(p); + Rendezvous::Args rendez_args; + rendez_args.device_context = device_context; + rendez_args.alloc_attrs = std::get<3>(p); + rendezvous->RecvAsync( - parsed, args, + parsed, rendez_args, [val, done, key, call_state](const Status& s, const Rendezvous::Args& send_args, const Rendezvous::Args& recv_args, diff --git a/tensorflow/core/common_runtime/rendezvous_util.h b/tensorflow/core/common_runtime/rendezvous_util.h index a54f8c3f94..3b6354603b 100644 --- a/tensorflow/core/common_runtime/rendezvous_util.h +++ b/tensorflow/core/common_runtime/rendezvous_util.h @@ -24,17 +24,25 @@ namespace tensorflow { typedef std::map NamedTensors; typedef std::function StatusCallback; -// Uses `rendezvous` to send tensors in `in`. -Status SendTensorsToRendezvous(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - gtl::ArraySlice tensors_to_send); - -void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - std::vector* received_tensors, - const StatusCallback& done); +// Uses `rendezvous` to send tensors in `tensors_to_send`. `device_context` +// should be the DeviceContext associated with the source of the tensors. +// `alloc_attrs` contains information about how the `tensors_to_send` are +// allocated. `alloc_attrs` should either be {} or should match the length of +// `keys`. +Status SendTensorsToRendezvous( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, gtl::ArraySlice tensors_to_send); + +// Uses `rendezvous` to obtain tensors. `device_context` should be the +// DeviceContext associated with the receiving device. `alloc_attrs` contains +// information as how to store the received tensors. Should be {} or match the +// length of `keys`. +void RecvOutputsFromRendezvousAsync( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, std::vector* received_tensors, + const StatusCallback& done); Status RecvOutputsFromRendezvous(Rendezvous* rendezvous, NamedTensors* out, const Rendezvous::Args& args); diff --git a/tensorflow/core/common_runtime/rendezvous_util_test.cc b/tensorflow/core/common_runtime/rendezvous_util_test.cc index 8ee9f4d522..093fa7921f 100644 --- a/tensorflow/core/common_runtime/rendezvous_util_test.cc +++ b/tensorflow/core/common_runtime/rendezvous_util_test.cc @@ -52,15 +52,14 @@ string MakeStringKey(const string& name) { TEST_F(RendezvousUtilTest, SendBeforeRecv) { // Fire off sends before receive the tensors. - Rendezvous::Args args; TF_ASSERT_OK(SendTensorsToRendezvous( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, {V("hello1"), V("hello2")})); Notification n; std::vector received_keys; RecvOutputsFromRendezvousAsync( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, &received_keys, [&n](const Status& status) { n.Notify(); }); n.WaitForNotification(); @@ -71,16 +70,14 @@ TEST_F(RendezvousUtilTest, SendBeforeRecv) { TEST_F(RendezvousUtilTest, RecvBeforeSend) { // Fire off recvs, wait for a notification in the callback. - Rendezvous::Args args; - Notification n; std::vector received_keys; RecvOutputsFromRendezvousAsync( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, &received_keys, [&n](const Status& status) { n.Notify(); }); TF_ASSERT_OK(SendTensorsToRendezvous( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, {V("hello1"), V("hello2")})); n.WaitForNotification(); diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc index 7a93b7406c..391ffda25c 100644 --- a/tensorflow/core/distributed_runtime/graph_mgr.cc +++ b/tensorflow/core/distributed_runtime/graph_mgr.cc @@ -337,8 +337,8 @@ Status GraphMgr::SendInputs(const int64 step_id, const NamedTensors& in) { keys.push_back(p.first); tensors_to_send.push_back(p.second); } - Status s = SendTensorsToRendezvous(rendezvous, Rendezvous::Args(), keys, - tensors_to_send); + Status s = + SendTensorsToRendezvous(rendezvous, nullptr, {}, keys, tensors_to_send); rendezvous->Unref(); return s; } @@ -362,7 +362,7 @@ void GraphMgr::RecvOutputsAsync(const int64 step_id, NamedTensors* out, received_keys->push_back(p.second); } RecvOutputsFromRendezvousAsync( - rendezvous, Rendezvous::Args(), keys, received_keys, + rendezvous, nullptr, {}, keys, received_keys, [done, rendezvous, received_keys, out, keys](const Status s) { rendezvous->Unref(); for (int i = 0; i < keys.size(); ++i) { @@ -420,8 +420,7 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id, keys.push_back(p.first); tensors_to_send.push_back(p.second); } - s = SendTensorsToRendezvous(rendezvous, Rendezvous::Args(), keys, - tensors_to_send); + s = SendTensorsToRendezvous(rendezvous, nullptr, {}, keys, tensors_to_send); } if (!s.ok()) { diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h index e8ae9aa74f..305b140a44 100644 --- a/tensorflow/core/framework/function.h +++ b/tensorflow/core/framework/function.h @@ -438,6 +438,16 @@ class FunctionLibraryRuntime { // Parameters for remote function execution. bool remote_execution = false; string source_device = ""; // Fully specified device name. + + // Allocator attributes specifying where the args are / rets should be put. + // These should either be {} or match the length of args / retvals. If {}, + // the default allocator attributes will be assumed for all args / retvals. + std::vector args_alloc_attrs; + std::vector rets_alloc_attrs; + + // If true, we create a new IntraProcessRendezvous, else use the existing + // one. + bool create_rendezvous = false; }; typedef std::function DoneCallback; virtual void Run(const Options& opts, Handle handle, diff --git a/tensorflow/core/kernels/function_ops.cc b/tensorflow/core/kernels/function_ops.cc index 1c6026c25d..f2290e87a5 100644 --- a/tensorflow/core/kernels/function_ops.cc +++ b/tensorflow/core/kernels/function_ops.cc @@ -328,9 +328,10 @@ class RemoteCallOp : public AsyncOpKernel { lib->Run(opts, handle, args, rets, [rets, done, ctx](const Status& status) { if (!status.ok()) { ctx->SetStatus(status); - } - for (size_t i = 0; i < rets->size(); ++i) { - ctx->set_output(i, (*rets)[i]); + } else { + for (size_t i = 0; i < rets->size(); ++i) { + ctx->set_output(i, (*rets)[i]); + } } delete rets; done(); -- GitLab From a6a61884396ef1d51b01f8e13df21becb23fd0c8 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Mon, 30 Oct 2017 22:26:51 -0700 Subject: [PATCH 557/573] eager: Documentation and example models. - Updated README - A preliminary "User's Guide" - A few example models, some with benchmarks PiperOrigin-RevId: 173996303 --- .../cudnn_rnn/python/layers/cudnn_rnn.py | 5 - tensorflow/contrib/eager/README.OPENSOURCE.md | 15 + tensorflow/contrib/eager/README.md | 87 +- .../contrib/eager/python/examples/BUILD | 15 + .../python/examples/linear_regression/BUILD | 25 + .../linear_regression/linear_regression.py | 157 +++ .../linear_regression_test.py | 119 +++ .../contrib/eager/python/examples/mnist/BUILD | 36 + .../eager/python/examples/mnist/README.md | 10 + .../eager/python/examples/mnist/mnist.py | 270 ++++++ .../python/examples/mnist/mnist_graph_test.py | 65 ++ .../eager/python/examples/mnist/mnist_test.py | 62 ++ .../python/examples/notebooks/1_basics.ipynb | 529 +++++++++++ .../examples/notebooks/2_gradients.ipynb | 864 +++++++++++++++++ .../examples/notebooks/3_datasets.ipynb | 218 +++++ .../eager/python/examples/resnet50/BUILD | 43 + .../eager/python/examples/resnet50/README.md | 34 + .../python/examples/resnet50/resnet50.py | 324 +++++++ .../examples/resnet50/resnet50_graph_test.py | 163 ++++ .../python/examples/resnet50/resnet50_test.py | 234 +++++ .../eager/python/examples/rnn_colorbot/BUILD | 26 + .../python/examples/rnn_colorbot/README.md | 26 + .../examples/rnn_colorbot/rnn_colorbot.py | 338 +++++++ .../rnn_colorbot/rnn_colorbot_test.py | 71 ++ .../eager/python/examples/rnn_ptb/BUILD | 35 + .../eager/python/examples/rnn_ptb/README.md | 42 + .../eager/python/examples/rnn_ptb/rnn_ptb.py | 348 +++++++ .../examples/rnn_ptb/rnn_ptb_graph_test.py | 164 ++++ .../python/examples/rnn_ptb/rnn_ptb_test.py | 154 +++ .../contrib/eager/python/g3doc/guide.md | 899 ++++++++++++++++++ tensorflow/tools/pip_package/BUILD | 1 + 31 files changed, 5362 insertions(+), 17 deletions(-) create mode 100644 tensorflow/contrib/eager/README.OPENSOURCE.md create mode 100644 tensorflow/contrib/eager/python/examples/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/linear_regression/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py create mode 100644 tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py create mode 100644 tensorflow/contrib/eager/python/examples/mnist/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/mnist/README.md create mode 100644 tensorflow/contrib/eager/python/examples/mnist/mnist.py create mode 100644 tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py create mode 100644 tensorflow/contrib/eager/python/examples/mnist/mnist_test.py create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/resnet50/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/resnet50/README.md create mode 100644 tensorflow/contrib/eager/python/examples/resnet50/resnet50.py create mode 100644 tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py create mode 100644 tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py create mode 100644 tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md create mode 100644 tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py create mode 100644 tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py create mode 100644 tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD create mode 100644 tensorflow/contrib/eager/python/examples/rnn_ptb/README.md create mode 100644 tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py create mode 100644 tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py create mode 100644 tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py create mode 100644 tensorflow/contrib/eager/python/g3doc/guide.md diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py index f6c206022c..3d3f8a3be0 100644 --- a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py +++ b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py @@ -18,7 +18,6 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.contrib.util import loader from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -26,12 +25,8 @@ from tensorflow.python.layers import base as base_layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import resource_loader from tensorflow.python.platform import tf_logging as logging -_cudnn_rnn_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_cudnn_rnn_ops.so")) - CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM diff --git a/tensorflow/contrib/eager/README.OPENSOURCE.md b/tensorflow/contrib/eager/README.OPENSOURCE.md new file mode 100644 index 0000000000..a4a3af08cf --- /dev/null +++ b/tensorflow/contrib/eager/README.OPENSOURCE.md @@ -0,0 +1,15 @@ +TensorFlow has many kernels for doing (deep) learning and data manipulation. +There are typically assembled into computational graphs which can run +efficiently in a variety of environments. + +We are exploring an alternative interaction, where kernels are invoked +immediately and call this "eager execution". We are hoping to retain the +benefits of graphs while improving usability with benefits like: + +- Immediate error messages and easier debugging +- Flexibility to use Python datastructures and control flow +- Reduced boilerplate + +Eager execution is under active development. +There are not many developer-facing materials yet, but stay tuned for updates +in this directory. diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md index a4a3af08cf..db11dbb0d7 100644 --- a/tensorflow/contrib/eager/README.md +++ b/tensorflow/contrib/eager/README.md @@ -1,15 +1,78 @@ -TensorFlow has many kernels for doing (deep) learning and data manipulation. -There are typically assembled into computational graphs which can run -efficiently in a variety of environments. +# TensorFlow Eager Execution -We are exploring an alternative interaction, where kernels are invoked -immediately and call this "eager execution". We are hoping to retain the -benefits of graphs while improving usability with benefits like: +> *WARNING*: This is a preview/pre-alpha version. The API and performance +> characteristics are subject to change. -- Immediate error messages and easier debugging -- Flexibility to use Python datastructures and control flow -- Reduced boilerplate +Eager execution is an experimental interface to TensorFlow that provides an +imperative programming style (à la [NumPy](http://www.numpy.org)). When you +enable eager execution, TensorFlow operations execute immediately; you do not +execute a pre-constructed graph with +[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session). -Eager execution is under active development. -There are not many developer-facing materials yet, but stay tuned for updates -in this directory. +For example, consider a simple computation in TensorFlow: + +```python +x = tf.placeholder(tf.float32, shape=[1, 1]) +m = tf.matmul(x, x) + +with tf.Session() as sess: + print(sess.run(m, feed_dict={x: [[2.]]})) + +# Will print [[4.]] +``` + +Eager execution makes this much simpler: + +```python +x = [[2.]] +m = tf.matmul(x, x) + +print(m) +``` + +## Caveats + +This feature is in early stages and work remains to be done in terms of smooth +support for distributed and multi-GPU training and CPU performance. + +- [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Aproj%3Aeager) +- Feedback is welcome, please consider + [filing an issue](https://github.com/tensorflow/tensorflow/issues/new) to provide it. + +## Installation + +Since eager execution is not yet part of a TensorFlow release, using it requires +either [building from source](https://www.tensorflow.org/install/install_sources) +or the latest nightly builds. The nightly builds are available as: + +- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and + +- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images. + +For example, to run the latest nightly docker image: + +```sh +# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker +nvidia-docker pull tensorflow/tensorflow:nightly-gpu +nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu + +# If you do not have a GPU, use the CPU-only image +docker pull tensorflow/tensorflow:nightly +docker run -it -p 8888:8888 tensorflow/tensorflow:nightly +``` + +And then visit http://localhost:8888 in your browser for a Jupyter notebook +environment. Try out the notebooks below. + +## Documentation + +For an introduction to eager execution in TensorFlow, see: + +- [User Guide](python/g3doc/guide.md) +- Notebook: [Basic Usage](python/examples/notebooks/1_basics.ipynb) +- Notebook: [Gradients](python/examples/notebooks/2_gradients.ipynb) +- Notebook: [Importing Data](python/examples/notebooks/3_datasets.ipynb) + +## Changelog + +- 2017/10/31: Initial preview release. diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD new file mode 100644 index 0000000000..aa21a6ab99 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/BUILD @@ -0,0 +1,15 @@ +# TensorFlow code for training gradient boosted trees. +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +py_library( + name = "examples_pip", + deps = [ + "//tensorflow/contrib/eager/python/examples/linear_regression", + "//tensorflow/contrib/eager/python/examples/mnist", + "//tensorflow/contrib/eager/python/examples/resnet50", + "//tensorflow/contrib/eager/python/examples/rnn_colorbot", + "//tensorflow/contrib/eager/python/examples/rnn_ptb", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD new file mode 100644 index 0000000000..bab7ad0c70 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD @@ -0,0 +1,25 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "linear_regression", + srcs = ["linear_regression.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + ], +) + +cuda_py_test( + name = "linear_regression_test", + size = "small", + srcs = ["linear_regression_test.py"], + additional_deps = [ + ":linear_regression", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py new file mode 100644 index 0000000000..d0130ebd11 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py @@ -0,0 +1,157 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""TensorFlow Eager Execution Example: Linear Regression. + +This example shows how to use TensorFlow Eager Execution to fit a simple linear +regression model using some synthesized data. Specifically, it illustrates how +to define the forward path of the linear model and the loss function, as well +as how to obtain the gradients of the loss function with respect to the +variables and update the variables with the gradients. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import sys + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe + + +class LinearModel(tfe.Network): + """A TensorFlow linear regression model. + + Uses TensorFlow's eager execution. + + For those familiar with TensorFlow graphs, notice the absence of + `tf.Session`. The `forward()` method here immediately executes and + returns output values. The `loss()` method immediately compares the + output of `forward()` with the target adn returns the MSE loss value. + The `fit()` performs gradient-descent training on the model's weights + and bias. + """ + + def __init__(self): + """Constructs a LinearModel object.""" + super(LinearModel, self).__init__() + self._hidden_layer = self.track_layer(tf.layers.Dense(1)) + + def call(self, xs): + """Invoke the linear model. + + Args: + xs: input features, as a tensor of size [batch_size, ndims]. + + Returns: + ys: the predictions of the linear mode, as a tensor of size [batch_size] + """ + return self._hidden_layer(xs) + + +def fit(model, dataset, optimizer, verbose=False, logdir=None): + """Fit the linear-regression model. + + Args: + model: The LinearModel to fit. + dataset: The tf.data.Dataset to use for training data. + optimizer: The TensorFlow Optimizer object to be used. + verbose: If true, will print out loss values at every iteration. + logdir: The directory in which summaries will be written for TensorBoard + (optional). + """ + + # The loss function to optimize. + def mean_square_loss(xs, ys): + return tf.reduce_mean(tf.square(model(xs) - ys)) + + loss_and_grads = tfe.implicit_value_and_gradients(mean_square_loss) + + tf.train.get_or_create_global_step() + if logdir: + # Support for TensorBoard summaries. Once training has started, use: + # tensorboard --logdir= + summary_writer = tf.contrib.summary.create_summary_file_writer(logdir) + + # Training loop. + for i, (xs, ys) in enumerate(tfe.Iterator(dataset)): + loss, grads = loss_and_grads(xs, ys) + if verbose: + print("Iteration %d: loss = %s" % (i, loss.numpy())) + + optimizer.apply_gradients(grads, global_step=tf.train.get_global_step()) + + if logdir: + with summary_writer.as_default(): + with tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar("loss", loss) + + +def synthetic_dataset(w, b, noise_level, batch_size, num_batches): + """tf.data.Dataset that yields synthetic data for linear regression.""" + + # w is a matrix with shape [N, M] + # b is a vector with shape [M] + # So: + # - Generate x's as vectors with shape [batch_size N] + # - y = tf.matmul(x, W) + b + noise + def batch(_): + x = tf.random_normal([batch_size, tf.shape(w)[0]]) + y = tf.matmul(x, w) + b + noise_level * tf.random_normal([]) + return x, y + + with tf.device("/device:CPU:0"): + return tf.data.Dataset.range(num_batches).map(batch) + + +def main(_): + tfe.enable_eager_execution() + # Ground-truth constants. + true_w = [[-2.0], [4.0], [1.0]] + true_b = [0.5] + noise_level = 0.01 + + # Training constants. + batch_size = 64 + learning_rate = 0.1 + + print("True w: %s" % true_w) + print("True b: %s\n" % true_b) + + model = LinearModel() + dataset = synthetic_dataset(true_w, true_b, noise_level, batch_size, 20) + + device = "gpu:0" if tfe.num_gpus() else "cpu:0" + print("Using device: %s" % device) + with tf.device(device): + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + fit(model, dataset, optimizer, verbose=True, logdir=FLAGS.logdir) + + print("\nAfter training: w = %s" % model.variables[0].numpy()) + print("\nAfter training: b = %s" % model.variables[1].numpy()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--logdir", + type=str, + default=None, + help="logdir in which TensorBoard summaries will be written (optional).") + FLAGS, unparsed = parser.parse_known_args() + + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py new file mode 100644 index 0000000000..39e7aabd7b --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py @@ -0,0 +1,119 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit tests for linear regression example under TensorFlow eager execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import glob +import os +import shutil +import tempfile +import time + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.contrib.eager.python.examples.linear_regression import linear_regression + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() > 0 else "/device:CPU:0" + + +class LinearRegressionTest(tf.test.TestCase): + + def setUp(self): + super(LinearRegressionTest, self).setUp() + self._tmp_logdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tmp_logdir) + super(LinearRegressionTest, self).tearDown() + + def testSyntheticDataset(self): + true_w = tf.random_uniform([3, 1]) + true_b = [1.0] + batch_size = 10 + num_batches = 2 + noise_level = 0. + dataset = linear_regression.synthetic_dataset(true_w, true_b, noise_level, + batch_size, num_batches) + + it = tfe.Iterator(dataset) + for _ in range(2): + (xs, ys) = it.next() + self.assertEqual((batch_size, 3), xs.shape) + self.assertEqual((batch_size, 1), ys.shape) + self.assertEqual(tf.float32, xs.dtype) + self.assertEqual(tf.float32, ys.dtype) + with self.assertRaises(StopIteration): + it.next() + + def testLinearRegression(self): + true_w = [[1.0], [-0.5], [2.0]] + true_b = [1.0] + + model = linear_regression.LinearModel() + dataset = linear_regression.synthetic_dataset( + true_w, true_b, noise_level=0., batch_size=64, num_batches=40) + + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) + linear_regression.fit(model, dataset, optimizer, logdir=self._tmp_logdir) + + self.assertAllClose(true_w, model.variables[0].numpy(), rtol=1e-2) + self.assertAllClose(true_b, model.variables[1].numpy(), rtol=1e-2) + self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) + + +class EagerLinearRegressionBenchmark(tf.test.Benchmark): + + def benchmarkEagerLinearRegression(self): + num_batches = 200 + batch_size = 64 + dataset = linear_regression.synthetic_dataset( + w=tf.random_uniform([3, 1]), + b=tf.random_uniform([1]), + noise_level=0.01, + batch_size=batch_size, + num_batches=num_batches) + burn_in_dataset = dataset.take(10) + + model = linear_regression.LinearModel() + + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) + + # Perform burn-in. + linear_regression.fit(model, burn_in_dataset, optimizer) + + start_time = time.time() + linear_regression.fit(model, dataset, optimizer) + wall_time = time.time() - start_time + + examples_per_sec = num_batches * batch_size / wall_time + self.report_benchmark( + name="eager_train_%s" % + ("gpu" if tfe.num_gpus() > 0 else "cpu"), + iters=num_batches, + extras={"examples_per_sec": examples_per_sec}, + wall_time=wall_time) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/mnist/BUILD b/tensorflow/contrib/eager/python/examples/mnist/BUILD new file mode 100644 index 0000000000..c61ec2dbae --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/BUILD @@ -0,0 +1,36 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "mnist", + srcs = ["mnist.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow/examples/tutorials/mnist:input_data", + ], +) + +cuda_py_test( + name = "mnist_test", + srcs = ["mnist_test.py"], + additional_deps = [ + ":mnist", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) + +cuda_py_test( + name = "mnist_graph_test", + srcs = ["mnist_graph_test.py"], + additional_deps = [ + ":mnist", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/mnist/README.md b/tensorflow/contrib/eager/python/examples/mnist/README.md new file mode 100644 index 0000000000..e987996b88 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/README.md @@ -0,0 +1,10 @@ +Classification model for the MNIST dataset using eager execution. + +To run: + +``` +python mnist.py +``` + +`mnist_graph_test.py` demonstrates that the same code that is executed eagerly +in `mnist.py` is used to construct a TensorFlow graph. diff --git a/tensorflow/contrib/eager/python/examples/mnist/mnist.py b/tensorflow/contrib/eager/python/examples/mnist/mnist.py new file mode 100644 index 0000000000..ae01bac0b5 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/mnist.py @@ -0,0 +1,270 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A deep MNIST classifier using convolutional layers. + +Sample usage: + python mnist.py --help +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import functools +import os +import sys +import time + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.examples.tutorials.mnist import input_data + +FLAGS = None + + +class MNISTModel(tfe.Network): + """MNIST Network. + + Network structure is equivalent to: + https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/examples/tutorials/mnist/mnist_deep.py + and + https://github.com/tensorflow/models/blob/master/tutorials/image/mnist/convolutional.py + + But written using the tf.layers API. + """ + + def __init__(self, data_format): + """Creates a model for classifying a hand-written digit. + + Args: + data_format: Either 'channels_first' or 'channels_last'. + 'channels_first' is typically faster on GPUs while 'channels_last' is + typically faster on CPUs. See + https://www.tensorflow.org/performance/performance_guide#data_formats + """ + super(MNISTModel, self).__init__(name='') + if data_format == 'channels_first': + self._input_shape = [-1, 1, 28, 28] + else: + assert data_format == 'channels_last' + self._input_shape = [-1, 28, 28, 1] + self.conv1 = self.track_layer( + tf.layers.Conv2D(32, 5, data_format=data_format, activation=tf.nn.relu)) + self.conv2 = self.track_layer( + tf.layers.Conv2D(64, 5, data_format=data_format, activation=tf.nn.relu)) + self.fc1 = self.track_layer(tf.layers.Dense(1024, activation=tf.nn.relu)) + self.fc2 = self.track_layer(tf.layers.Dense(10)) + self.dropout = self.track_layer(tf.layers.Dropout(0.5)) + self.max_pool2d = self.track_layer( + tf.layers.MaxPooling2D( + (2, 2), (2, 2), padding='SAME', data_format=data_format)) + + def call(self, inputs, training): + """Computes labels from inputs. + + Users should invoke __call__ to run the network, which delegates to this + method (and not call this method directly). + + Args: + inputs: A batch of images as a Tensor with shape [batch_size, 784]. + training: True if invoked in the context of training (causing dropout to + be applied). False otherwise. + + Returns: + A Tensor with shape [batch_size, 10] containing the predicted logits + for each image in the batch, for each of the 10 classes. + """ + + x = tf.reshape(inputs, self._input_shape) + x = self.conv1(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.max_pool2d(x) + x = tf.layers.flatten(x) + x = self.fc1(x) + if training: + x = self.dropout(x) + x = self.fc2(x) + return x + + +def loss(predictions, labels): + return tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + logits=predictions, labels=labels)) + + +def compute_accuracy(predictions, labels): + return tf.reduce_sum( + tf.cast( + tf.equal( + tf.argmax(predictions, axis=1, + output_type=tf.int64), + tf.argmax(labels, axis=1, + output_type=tf.int64)), + dtype=tf.float32)) / float(predictions.shape[0].value) + + +def train_one_epoch(model, optimizer, dataset, log_interval=None): + """Trains model on `dataset` using `optimizer`.""" + + tf.train.get_or_create_global_step() + + def model_loss(labels, images): + prediction = model(images, training=True) + loss_value = loss(prediction, labels) + tf.contrib.summary.scalar('loss', loss_value) + tf.contrib.summary.scalar('accuracy', + compute_accuracy(prediction, labels)) + return loss_value + + for (batch, (images, labels)) in enumerate(tfe.Iterator(dataset)): + with tf.contrib.summary.record_summaries_every_n_global_steps(10): + batch_model_loss = functools.partial(model_loss, labels, images) + optimizer.minimize( + batch_model_loss, global_step=tf.train.get_global_step()) + if log_interval and batch % log_interval == 0: + print('Batch #%d\tLoss: %.6f' % (batch, batch_model_loss())) + + +def test(model, dataset): + """Perform an evaluation of `model` on the examples from `dataset`.""" + avg_loss = tfe.metrics.Mean('loss') + accuracy = tfe.metrics.Accuracy('accuracy') + + for (images, labels) in tfe.Iterator(dataset): + predictions = model(images, training=False) + avg_loss(loss(predictions, labels)) + accuracy(tf.argmax(predictions, axis=1, output_type=tf.int64), + tf.argmax(labels, axis=1, output_type=tf.int64)) + print('Test set: Average loss: %.4f, Accuracy: %4f%%\n' % + (avg_loss.result(), 100 * accuracy.result())) + with tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar('loss', avg_loss.result()) + tf.contrib.summary.scalar('accuracy', accuracy.result()) + + +def load_data(data_dir): + """Returns training and test tf.data.Dataset objects.""" + data = input_data.read_data_sets(data_dir, one_hot=True) + train_ds = tf.data.Dataset.from_tensor_slices((data.train.images, + data.train.labels)) + test_ds = tf.data.Dataset.from_tensors((data.test.images, data.test.labels)) + return (train_ds, test_ds) + + +def main(_): + tfe.enable_eager_execution() + + (device, data_format) = ('/gpu:0', 'channels_first') + if FLAGS.no_gpu or tfe.num_gpus() <= 0: + (device, data_format) = ('/cpu:0', 'channels_last') + print('Using device %s, and data format %s.' % (device, data_format)) + + # Load the datasets + (train_ds, test_ds) = load_data(FLAGS.data_dir) + train_ds = train_ds.shuffle(60000).batch(FLAGS.batch_size) + + # Create the model and optimizer + model = MNISTModel(data_format) + optimizer = tf.train.MomentumOptimizer(FLAGS.lr, FLAGS.momentum) + + if FLAGS.output_dir: + train_dir = os.path.join(FLAGS.output_dir, 'train') + test_dir = os.path.join(FLAGS.output_dir, 'eval') + tf.gfile.MakeDirs(FLAGS.output_dir) + else: + train_dir = None + test_dir = None + summary_writer = tf.contrib.summary.create_summary_file_writer( + train_dir, flush_secs=10) + test_summary_writer = tf.contrib.summary.create_summary_file_writer( + test_dir, flush_secs=10, name='test') + checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir, 'ckpt') + + with tf.device(device): + for epoch in range(1, 11): + with tfe.restore_variables_on_create( + tf.train.latest_checkpoint(FLAGS.checkpoint_dir)): + global_step = tf.train.get_or_create_global_step() + start = time.time() + with summary_writer.as_default(): + train_one_epoch(model, optimizer, train_ds, FLAGS.log_interval) + end = time.time() + print('\nTrain time for epoch #%d (global step %d): %f' % ( + epoch, global_step.numpy(), end - start)) + with test_summary_writer.as_default(): + test(model, test_ds) + all_variables = ( + model.variables + + tfe.get_optimizer_variables(optimizer) + + [global_step]) + tfe.Saver(all_variables).save( + checkpoint_prefix, global_step=global_step) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--data-dir', + type=str, + default='/tmp/tensorflow/mnist/input_data', + help='Directory for storing input data') + parser.add_argument( + '--batch-size', + type=int, + default=64, + metavar='N', + help='input batch size for training (default: 64)') + parser.add_argument( + '--log-interval', + type=int, + default=10, + metavar='N', + help='how many batches to wait before logging training status') + parser.add_argument( + '--output_dir', + type=str, + default=None, + metavar='N', + help='Directory to write TensorBoard summaries') + parser.add_argument( + '--checkpoint_dir', + type=str, + default='/tmp/tensorflow/mnist/checkpoints/', + metavar='N', + help='Directory to save checkpoints in (once per epoch)') + parser.add_argument( + '--lr', + type=float, + default=0.01, + metavar='LR', + help='learning rate (default: 0.01)') + parser.add_argument( + '--momentum', + type=float, + default=0.5, + metavar='M', + help='SGD momentum (default: 0.5)') + parser.add_argument( + '--no-gpu', + action='store_true', + default=False, + help='disables GPU usage even if a GPU is available') + + FLAGS, unparsed = parser.parse_known_args() + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py b/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py new file mode 100644 index 0000000000..1af2655312 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py @@ -0,0 +1,65 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib.eager.python.examples.mnist import mnist + + +def data_format(): + return "channels_first" if tf.test.is_gpu_available() else "channels_last" + + +class MNISTGraphTest(tf.test.TestCase): + + def testTrainGraph(self): + # The MNISTModel class can be executed eagerly (as in mnist.py and + # mnist_test.py) and also be used to construct a TensorFlow graph, which is + # then trained in a session. + with tf.Graph().as_default(): + # Generate some random data. + batch_size = 64 + images = np.random.randn(batch_size, 784).astype(np.float32) + digits = np.random.randint(low=0, high=10, size=batch_size) + labels = np.zeros((batch_size, 10)) + labels[np.arange(batch_size), digits] = 1. + + # Create a model, optimizer, and dataset as would be done + # for eager execution as well. + model = mnist.MNISTModel(data_format()) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + dataset = tf.data.Dataset.from_tensors((images, labels)) + + # Define the loss tensor (as opposed to a loss function when + # using eager execution). + (images, labels) = dataset.make_one_shot_iterator().get_next() + predictions = model(images, training=True) + loss = mnist.loss(predictions, labels) + + train_op = optimizer.minimize(loss) + init = tf.global_variables_initializer() + with tf.Session() as sess: + # Variables have to be initialized in the session. + sess.run(init) + # Train using the optimizer. + sess.run(train_op) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/mnist/mnist_test.py b/tensorflow/contrib/eager/python/examples/mnist/mnist_test.py new file mode 100644 index 0000000000..205709fe2e --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/mnist_test.py @@ -0,0 +1,62 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.contrib.eager.python.examples.mnist import mnist + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" + + +def data_format(): + return "channels_first" if tfe.num_gpus() else "channels_last" + + +def random_dataset(): + batch_size = 64 + images = tf.random_normal([batch_size, 784]) + digits = tf.random_uniform([batch_size], minval=0, maxval=10, dtype=tf.int32) + labels = tf.one_hot(digits, 10) + return tf.data.Dataset.from_tensors((images, labels)) + + +class MNISTTest(tf.test.TestCase): + + def testTrainOneEpoch(self): + model = mnist.MNISTModel(data_format()) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + dataset = random_dataset() + with tf.device(device()): + tf.train.get_or_create_global_step() + mnist.train_one_epoch(model, optimizer, dataset) + + def testTest(self): + model = mnist.MNISTModel(data_format()) + dataset = random_dataset() + with tf.device(device()): + tf.train.get_or_create_global_step() + mnist.test(model, dataset) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb new file mode 100644 index 0000000000..01616f2e7d --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb @@ -0,0 +1,529 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U9i2Dsh-ziXr" + }, + "source": [ + "# Eager Execution Tutorial: Basics\n", + "\n", + "This notebook introduces the basics of using TensorFlow's eager execution capabilities. It covers concepts such as:\n", + "\n", + "* Importing required packages\n", + "* Enabling eager execution\n", + "* Creating and using TensorFlow Tensors and Variables\n", + "* Using TensorFlow interactively\n", + "* Using GPUs with eager execution enabled\n", + "\n", + "This notebook does *not* cover modeling topics, such as gradients." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z1JcS5iBXMRO" + }, + "source": [ + "# Step 1: Import Eager\n", + "\n", + "The key imports for eager execution are the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RlIWhyeLoYnG" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "import tensorflow.contrib.eager as tfe" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H9UySOPLXdaw" + }, + "source": [ + "# Step 2: Enable eager execution\n", + "\n", + "All future TensorFlow calls will execute the\n", + "underlying TensorFlow ops immediately:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "WPTUfGq6kJ5w" + }, + "outputs": [], + "source": [ + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "twBfWd5xyu_d" + }, + "source": [ + "# Step 3: Interactively Use TensorFlow!\n", + "\n", + "Now you can call TensorFlow functions and get results, immediately! No more `tf.Sessions`!\n", + "\n", + "TensorFlow will automatically wrap native Python types for you with operator overloading for TensorFlow Tensors." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ngUe237Wt48W" + }, + "outputs": [], + "source": [ + "print(tf.add(1, 2))\n", + "print(tf.add([1, 2], [3, 4]))\n", + "print(tf.square(5))\n", + "print(tf.reduce_sum([1, 2, 3]))\n", + "print(tf.encode_base64(\"hello world\"))\n", + "print(\"\")\n", + "\n", + "x = tf.constant(2)\n", + "y = tf.constant(3)\n", + "print(x * y + 1)\n", + "\n", + "# Most TensorFlow ops are directly usable with eager execution, giving\n", + "# results immediately.\n", + "print(tf.contrib.signal.hamming_window(x * y + 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IDY4WsYRhP81" + }, + "source": [ + "Numpy arrays are supported, too:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "lCUWzso6mbqR" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "ones = np.ones([3, 3])\n", + "\n", + "print(\"numpy 3x3 matrix of 1s:\")\n", + "print(ones)\n", + "print(\"\")\n", + "\n", + "print(\"Multiplied by 42:\")\n", + "print(tf.multiply(ones, 42))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PBNP8yTRfu_X" + }, + "source": [ + "# Step 4: Define and Print TensorFlow Variables\n", + "\n", + "To define TensorFlow variables, use the `get_variable()` function as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "3Twf_Rw-gQFM" + }, + "outputs": [], + "source": [ + "x = tf.get_variable(name=\"x\", shape=[], dtype=tf.float32, initializer=tf.zeros_initializer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "45G7094TxsMb" + }, + "source": [ + "## Printing TensorFlow Variables" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "UJBJeZ5XxuwA" + }, + "outputs": [], + "source": [ + "# This does NOT print the Variable's actual value:\n", + "print(\"Printing a TensorFlow Variable:\")\n", + "print(x)\n", + "print(\"\")\n", + "\n", + "# A TensorFlow variable represents a reference to a tensor.\n", + "# The `read_value()` method provides access to the current value of the\n", + "# variable. Tensorflow Variables are automatically initialized according to the\n", + "# semantics defined in tf.get_variable().\n", + "print(\"Printing a TensorFlow Variable's value using .read_value():\")\n", + "print(x.read_value())\n", + "print(\"\")\n", + "\n", + "print(\"Printing a TensorFlow Variable's value using .read_value().numpy():\")\n", + "print(x.read_value().numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "2njjWHcTpBEn" + }, + "source": [ + "## Changing a TensorFlow Variable's value\n", + "\n", + "To change a TensorFlow Variable's value, use its `.assign()` or `.assign_add()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "v3wr6Erbo_hB" + }, + "outputs": [], + "source": [ + "x.assign(42)\n", + "print(x.read_value())\n", + "\n", + "x.assign_add(3)\n", + "print(x.read_value())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "uhtynjHVpTB5" + }, + "source": [ + "## Use a Variable just like any other Tensor" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "7PbktdnHoehR" + }, + "outputs": [], + "source": [ + "print(x + 3)\n", + "\n", + "# This code will broadcast the value across the list of numbers:\n", + "print(x * [1, 2, 4])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GVChqwlwy1SI" + }, + "source": [ + "# Step 5: Debug Errors with Instant Feedback\n", + "\n", + "TensorFlow's eager execution helps you identify and debug runtime issues through interactive exploration of code snippets.\n", + "\n", + "Below, we'll define a length-4 vector, and attempt two `tf.slice()` operations,\n", + "one being legal and the other being illegal, leading to a runtime error that is\n", + "raised immediately." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "23ap04N0v4k0" + }, + "outputs": [], + "source": [ + "vector = tf.constant([10.0, 20.0, 30.0, 40.0])" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "FCUMsIYxxRRa" + }, + "outputs": [], + "source": [ + "# Works, because the values of `begin` and `size` (the 2nd and 3rd input\n", + "# arguments) are within the bound of `vector`.\n", + "print(tf.slice(vector, [1], [3]))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "T8me2oCNxpFp" + }, + "outputs": [], + "source": [ + "# The following does NOT work, because the value of `size` (the 3rd\n", + "# argument) causes the indices to go out of the bounds of `vector`. The\n", + "# error is raised immediately.\n", + "try:\n", + " print(tf.slice(vector, [1], [4]))\n", + "except tf.OpError as e:\n", + " print(\"Caught error: %s\" % e)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "irxJhAgar84v" + }, + "source": [ + "# Step 6: Using the GPU\n", + "\n", + "You can place Tensors on the GPU by calling a Tensor's `.gpu()` method.\n", + "\n", + "The first operation executing on the GPU may be slow as TensorFlow initializes. Subsequent uses will be much faster." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "7J4N9baqaKCL" + }, + "outputs": [], + "source": [ + "# The example code from here on will work only if your notebook\n", + "# is running on a machine with a functional CUDA GPU. The following\n", + "# line checks that.\n", + "is_gpu_available = tfe.num_gpus() \u003e 0\n", + "\n", + "# Create some Tensors\n", + "SIZE = 1000\n", + "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", + "\n", + "if is_gpu_available:\n", + " gpu_tensor = cpu_tensor.gpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "4E-2n7VbzY1n" + }, + "outputs": [], + "source": [ + "# Time a CPU-based matrix multiplication\n", + "\n", + "print(\"Time to conduct matmul on CPU:\")\n", + "%time tf.matmul(cpu_tensor, cpu_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "vbSFW-T5zhZF" + }, + "outputs": [], + "source": [ + "# Time GPU-based matrix multiplications.\n", + "\n", + "if is_gpu_available:\n", + " # First use of the GPU will be slow:\n", + " print(\"Time to conduct first matmul on GPU:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)\n", + " print()\n", + "\n", + " # Subsequent uses are much faster:\n", + " print(\"Time to conduct second matmul on GPU:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "E5pIOe3Rz7iW" + }, + "outputs": [], + "source": [ + "# Second timing demo for GPUs, after it has been used once:\n", + "\n", + "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", + "print(\"Time to conduct CPU matmul:\")\n", + "%time tf.matmul(cpu_tensor, cpu_tensor)\n", + "print()\n", + "\n", + "if is_gpu_available:\n", + " gpu_tensor = cpu_tensor.gpu()\n", + " print(\"Time to conduct GPU matmul:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "name": "Eager Execution Tutorial: Basics", + "provenance": [ + { + "file_id": "0B0kLcpwLFwKEVm9XNkFueGk4bTg", + "timestamp": 1504118841551 + } + ], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb new file mode 100644 index 0000000000..3b7e2cd435 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb @@ -0,0 +1,864 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "vDJ4XzMqodTy" + }, + "source": [ + "# Eager Execution: Working with Gradients\n", + "\n", + "This notebook demonstrates:\n", + "\n", + "* How to get gradients using TensorFlow's eager execution capabilities\n", + "* How to apply the gradients so you can update your variables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GQJysDM__Qb0" + }, + "source": [ + "# Setup: Import eager and enable eager execution.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "OiMPZStlibBv" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "import tensorflow.contrib.eager as tfe\n", + "\n", + "# Enable eager execution.\n", + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1CLWJl0QliB0" + }, + "source": [ + "# Fitting a Simple Linear Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-39gouo7mtgu" + }, + "source": [ + "## Step 1: Synthesize some data\n", + "\n", + "To demonstrate fitting a model with TensorFlow's eager execution, we'll fit a linear model to some synthesized data (which includes some noise).\n", + "\n", + "In the code, we use the variable names `w` and `b` to represent the single weight and bias we'll use to fit our model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "rQsdCg9PfIL-" + }, + "outputs": [], + "source": [ + "# The constants we'll try to fit our variables to:\n", + "true_w = 3\n", + "true_b = 2\n", + "\n", + "NUM_EXAMPLES = 1000\n", + "\n", + "# Our inputs:\n", + "inputs = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", + "\n", + "# Our labels, with noise:\n", + "noise = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", + "labels = inputs * true_w + true_b + noise" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 360, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 127, + "status": "ok", + "timestamp": 1505502830690, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "O4lsC4ckAcar", + "outputId": "2f760690-cafb-4777-b970-91d839f99faf" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAFXCAYAAACC+2avAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt8VPWd99+TK7kykxtJQIebqZfaqogtrhKNa1ooEKl9\nCrpVn9ZNW6x9VWsbCi7aVUt01NZ9tq21KVZlFey2YkQNohhj3QWK2liCF5RIBCc3yEwmIZnMTOY8\nf/zmzJwzSSBAYibh+369eIU5c87vXLh8zvdu0TRNQxAEQRCEmCVurC9AEARBEISjI2ItCIIgCDGO\niLUgCIIgxDgi1oIgCIIQ44hYC4IgCEKMI2ItCIIgCDHOiIj16tWrufjii1m8eHF4269//Wvmz5/P\n0qVLWbp0Ka+//vpInEoQBEEQTjksI1Fn/eabb5KWlkZFRQWbN28GlFinpaXx7W9/+6QvUhAEQRBO\nZUbEsr7wwgvJzMwcsF36rQiCIAjCyTOqMesnn3ySsrIybr/9drq6ukbzVIIgCIIwYRk1sb722mt5\n5ZVXqK6uJicnh8rKytE6lSAIgiBMaEZNrLOysrBYLAB885vfZPfu3cc8RtzmgiAIgjCQhJFaKFpo\n29vbyc3NBeDll1+mqKjomGtYLBba2yeuuzw3N0Pubxwzke9vIt8byP2Nd06F+zsWIyLWt912Gzt3\n7sTtdnPZZZfxwx/+kJ07d/Lee+8RFxfH1KlTueuuu0biVIIgCIJwyjEiYv3ggw8O2Hb11VePxNKC\nIAiCcMojHcwEQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBOHXo6HCzcmUt\nTU2Z2O2dOBwl2GzWsb6smEfEWhAEQfjMWLmylurq6wAL9fUasJ6qqqVjfVkxj7jBBUEQhM+MpqZM\nwBL6ZAl9Fo6FiLUgCILwmWG3dwJa6JOG3e4Zy8sZN4gbXBAEQfjMcDhKgPWhmLUHh+Pysb6kcYGI\ntSAIgvCZYbNZJUZ9AogbXBAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFr\nQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhx\nRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBODE6OtysXFmL02mjsLADh6MEm806rGOamjKx2zuHdYww\n9oyIWK9evZrXXnuN7OxsNm/eDEBnZye33norn376KdOmTeOhhx4iIyNjJE4nCIIgACtX1lJdfR1g\nATRgPVVVS037RIuzz9dDTc33AQv19YMfI8QeI+IG//rXv866detM237/+98zb948XnrpJb70pS/x\nyCOPjMSpBEEQhBBNTZkooQawhD6b0QW9vv4qqquvZ/v27mMeI8QeIyLWF154IZmZ5j/wbdu2sXSp\neltbunQpr7zyykicShAEQQhht3eiLGoADbvdM2CfaEGH7GMeI8Qeoxaz7ujoICcnB4Dc3FxcLtdo\nnUoQBOGUxOEoAdaHYtYuHI7LAbPru61tD1AM2AAXkyY5sVr/CBxi3rwMHI5FY3cDwrCJuQSz3NyJ\nHdeW+xvfTOT7m8j3BhPz/uLi+klOTgQgOTmBnJwMsrIyuPnm5w2x7DKmTbuPgoJzaG7ew8GDt6PH\nuDMyNpKdncFNNz3Pxx+nM2NGFw8/vJCsrNhLOJuIf37Hw6iJdXZ2NocOHSInJ4f29naysrKGdVx7\ne9doXdKYk5ubIfc3jpnI9zeR7w0m7v2Vlz8XFuVduzT6+lSy2N69KRhd3zk5Z/LCC5dRWtrPwYOR\n7Xv3pnDjjYOvEUtM1D8/neG8iIxYnbWmaabPJSUlPPPMMwBs2rSJK664YqROJQiCIDB0gtlQsezB\ntg8nSU0Ye0bEsr7tttvYuXMnbrebyy67jB/+8Id897vf5Uc/+hF/+ctfKCws5D/+4z9G4lSCIAhC\nCLu9M1R+pdzauijrsWxVruUJx7JXrZrDrl2VuFzTsNkOsnr1EtaufWvQNYTYYkTE+sEHHxx0+2OP\nPTYSywuCIAiDMFSCmc1mHdSVXVn5Nk7nKsBCb6/G2rXrhxR2IbaIuQQzQRAEYXjoojxYTHewTmWD\nubyHEnYhthCxFgRBmIAYu5vpncrsdk1c3uMUEWtBEIQYYai+3SfSz3swK/rpp+cgLu/xiYi1IAhC\njDCYNVxVtXTI7UdjsOQzcXmPX0SsBUEQYoShyqhOpLxKEscmFiLWgiAIMcJQpVjm7S7a2t6ltJSw\nS3ywphojYUXLOM3YQcRaEAQhRhjKGjZub2t7F6dzFU6nconX1T1AaelU7r770mEL6XBF+ETc78Lo\nIGItCIIQIwxlDRu3l5aC0xlxibvdZ/KnPy06rjahwxVh6W4WO4xYu1FBEARh9IluGQpqPvXxCOlw\nRXg4IziFzwaxrAVBEMYRuku8ttaPx5MCLAQ0CgoODXuNoWLjQ51LktTGHhFrQRCEcYTuEr/hhv+i\npiYBeBY4xNtvu3G53ANiz4PFp4crwlLqFTuIWAuCIIxDmpsLgF5gOWChtVWjomJg7Hmo+LSI8PhC\nxFoQBGEcEG0hFxT4qK+fwrFiz5IkNjGQBDNBEITPiI4ON+Xlmygt3UZ5+TO4XO5hf69byPX1V1Fd\nfT0QoLBwN8dKAJMksYmBWNaCIAifEdEu6V27KqmtvS4cZz5aSVW0hdzcXEBt7SIqKgaOyDQiSWIT\nAxFrQRCEz4howXU6P09FRe2Qgmx0WRcUNFNf/xSQAXgoKPAcdUSmznCTxKRbWWwjbnBBEIRBOJbL\n+kQwu6RdwLts3Up4/aO7rBOBa4DFwLWhzyNHtJu9oqJ2RNcXTg6xrAVBEAZhNFptOhwl7NpVidP5\neeBdYCW9vRaqq9X6DkcJfX3r2LEjDjiMz5cWLsdqbs7B7AbPOalriUYS0WIbsawFQRAG4XjFaziW\nuM1mpbb2OsrK3KSkFA5Y32azkpychNv9bdzun1JTsyJs4UZb3QUFLeHzLVv21Elb/pKIFtuIZS0I\ngjAIw+3ypROxxDupr3+RurqXKS6OHxD71WPI5eXPhCxq8/pDvSREJ4r5fAkmy/94eoMb0WPV+/Yl\nUFhYSXZ2ETNn9kgiWowhYi0IgjAIx5tFHRHZGmABbvcWqqvT2LXrCWprrx+QrOVwlODzPcL27V1A\nNj5ffzhuPdhLQnSiWGnpNoZr+R8teczo7geNuXNlslYsImItCIIwCMfbajMisunAFvTOYk7n4kE7\ni9lsVpKSUnG7vwdYqKnRSEpaf9SXBKPotrXtAcoYjuV/PCVhEquOTUSsBUEQBmEoa3So7brI1tW1\n4HafyXAEcDChPNpLgtkKLqawsJK8vLMpKurl7ruHtvyPJsjH6+4XxgYRa0EQhEEYyhodarsusi6X\nm8svfwKnczFDCaAu+Pv3t6CSuoYnlGbRtZGXdzZbt15Bbm4GH3xwgPLyTYO6uo8myNI0ZXwgYi0I\nQswylo06IsLoBmrC9dD79iUQbaV2dLi55ZaXQiVXh5gzJ5kvfnEdzc052O0eVq26wCSkPl8PNTXf\nBzqBDVitXoqLE44plEcT3aO5uo8myDJZa3wgYi0IQswyGrXOwyXSMcwJ3Bauhy4srCTaGl65spYt\nW24Mb9u2bQNlZQG2br0CgPLyTab7sFofCO1rBa5l+vRnqaq6Ilz+NdTLiS66+/bF09HRRGNjEeXl\nz/Doo2VHdXWLII9/RKwFQYhZxjb5Se8Y9rzpGrKzi5g7V1mpBQUt+HwJvPZaErABWIgS4AyamvrD\nK0XfB2SjOphtAdJoa9uDyzXnmC8nuuhef/3TNDSswum0sHu3xo03PoHdjsSeJzAi1oIgxCxjmfwU\n6RjWhdGSnjmzJyygRotZ7fMgUAD00NbWTmkphnGWkTXmzQvyzjsP43SuwpgxPtyXE+Vuj+xXV6ex\nY8cVSOx54iJiLQhCzDKWyU+RF4WFDBVXHmgxfw5YRHLy7TidP8XptFFfr7Fgwe8oKzPex1dYtuwt\nnM7IsVu3gs023HKsQxhfIOCQuLonOCLWgiDELCMtQEdLWIv+bvXqOUReFAI4HFcOSG6LtvyhO/T7\n01Eu7nSgiwMHMnn11SVHPba3N5He3psoLKwkK6uIjo697Ntnp7z8mQGx63nz0qmp2YCawNXF/Pky\nHWuiI2ItCMIpw2Ax4fvuu5yVK2upqwvgdicDl1FfP5nBktmGEvTaWj8eTwrKCteAg4BqdgIaHR2V\nA65F9xps3Qq9vX7Uf8dv0NOTwFlnfUJDw3SczgwaGjz4fM/z+OPfCl8DJGG1eoGDzJuXwaOPXkN/\n/4BTCBMIEWtBEE4ZBosJR7fbhI3ANYPGi4dKALvhhv+ipkYD/gDk4PMlocqyAGpwuQoHWMjmHuEp\nqGQ2C273Il5//U7g1vA1bd/+AKCEuqRkfTjWDarrWVaWdch51sLEQMRaEIRThsES1gbGndOJjhfr\nFvXWrRj27WTz5oMUFf03gcCh0PbbAQuapqGywy3ActMYzGhr3eEooa7uZdzuyDX090+PuqZsQL0s\nqPGa0h70VEPEWhCEU4bBEtYqKl41CbjV+j7FxS5TIlnEot5AJLHrRYLBVSGR1YDHMQusDzWF+OjC\narNZ+fKX+9myJXINOTkHaWszZ4+D7hno5ni6ngkTAxFrQRBOGWw2azhG3dSUSUXFq1GJZB4cjuUD\nEsn27YtHucctwL1YLFPQNLMQQzvmDG0L8Klp2/vvv0lJyRFmzQqYXOIWSwD1IqASxs49N430dHP2\nOOiegSWha0mjsLABh+O6UXteQuwgYi0IQkwQnby1atUcKivfHvFWoyfSFa2jowmIxImTk9fg9Z6F\nWZyt6CIKO1BW9W3AfcDZwBG83ttoaNhCQ8P1pvM2NxcAV4XPd/jws2zYcMWA61Cegc2hZ+LG4bgO\nTYNlyzawd2/KZ96SVfjsELEWBCEmiBbRXbsqw4lUx9NqdLDyrNzcjPD3J9IVbfLkqTidG9FLsU47\nrZDZsz1s3/4A3d2ZBAKTQmumAW8CPwXeAGzAOcBiw2rpA8473OYvg5WyRbcy/SxbsgqfHSLWgiDE\nBNEi6nJN40QSqQaznJ999vrw98cSxsHEvrPzU4yW9ZEjlfzqV9excmUtjY2pHD78AR5PBt3de1Ad\nzGxEOp8ZO6C5gJ1AO3v2OLnhBicPPbT4pJq/yDzqUwMRa0EQYoJoEbXZDtLbG/mcn38oPOSioKAZ\nSAxNtTK7fo8lXqtWzWHnzntoa8sjPv4Q3d3puFzu8PGDiX12dpGp21h2dtGAkq/k5DXARcAelCir\nzmeZmR34fHfg9Z4P7AXuBiz4/VqosckLJCWlnrC7X+ZRnxqIWAuCMGocz4jLaOty9eolrF0b+ezz\n+amuVpOt1DSsaxisucn+/QHgSeBrwOQB4lVZ+TYtLbOAawgGLWzbplFREXEdDyb2M2d2snu3uT94\n9H59fRcBS1Au7/tISSmktBQcjjKWLXuL+vqrgM2mYyCD7ds/xe3+HsNxYw/2PB2OEpKTN4Zi1tIT\nfKIiYi0IwqhxPMlcg8Vjq6rs4d+Xlm4jInQZGEVv61bYtesJnM6bUC5oNYayuHjKAPFSmd1O1DSt\nLmAhdXUBGhubqKx8m/37W4gujVq1ag67dlXick3DZjvA6tVl3HnndswJZkfC1wNTuOyyI1RVqa5j\nEeu3K+qYLlQN9fDc2EM9z6efvkaaokxwRKwFQRg1oq3PfftSB8xr1jSGZX2b3b0ejKKn+mqvRu8+\nBhamTz+DqqqBGdXRmd2wAbd7Epde+if8/n9HdR4zD+6oqKgNJ7v19mosXVpJd7debuUDmoHvh86g\nAclApP+ncQ71oUP30NMzlbi4w8yblw4khLqfHduNLfHpUxcRa0EQRo3oeGpHx14aGswZ3sCg1uJQ\nfbhVL20f8ERo3URgAZFsbDia6EXHn5XYXoXfHwh9tqLizVU0NZ1BRcWrNDamYRTJSBexxSjX9lVA\nDSrT+wPgX2lufi18zqMNJHG53CQlDS+5TOLTpy4i1oIgjBrRceh9++wGoeykrq6Vvr4pKAt1IWAN\nW4tDuXxVL20Vu1ax6eXo4lVY2EBeXvCoojdz5pFQ/LkTeDG09QXgI4zdydzun1Bfr85dWLiWgS5v\njYgrezLqheFFIAd4gYKCwYV0sLjzcEutxnJkqDC2iFgLgjBqRFuU5eXP0NBgFkTzAI3lYWtxKJev\nUbCUIK4LZYV7cDiuO2YmtX58bW0LHs9PDedfh8redtHb68bvj8S0s7KmM3euOmdb27s4nStQrvh7\ngQySklbS359Mf/9cVDvQBcBfBj3/8cTxT0bYhYmFiLUgCJ8ZRqHdv99rGl6RkuKntHR92FocyuUb\n/QJgFLSKilePWfqkH19auo36eqM73A/00tX1KZr2C4wx7Vmz+sOu+VtvPURPzyaOHPkYv//HgA2f\nL5Kdrr94NDfnDCq2xxN3PpFua8LERMRaEITPDKPQKnd2RIxLSzEJ0VAu32gB7Orq5NVXf4guaD7f\nOh5/fNmAc+vH7dsXT0dHE93d8Zhd25OBa9G05zCKqdXqxeG4ElDiWVNzI2ZvwDVEZ6dDGna7e1Cx\ntdu1YcedJaFM0BGxFgRhTDhW/HWopKxoAUxMXItR0LZvjxtwzOHDxjnQG1HZ4CrrOzPTS3d3C8Hg\nTaG9zVOtiosThmy4EkloM2enT5q0i9Wrl/G9731EtNg+/XT04BBJKBOOjYi1IAhjwtEypI9GtGD2\n9+dgtpAPDzjmpptqQhncnahJWJF4dFzcM+Tnazidk0N7LwDuwGqdQXFxAqtWXRAuN2tr2wMUo2q5\nXUyatAuLxU1m5l407S7a2s5HDez4MZdc8is0bSrwGCpbXDVoOZ77loQyQUfEWhCEMWG43c2i9yso\n8Jmszby8Vlpa9PGSrfT2urDbN2GzHWDTpjJmzLDz8cdqAIfK1r4NYzwaDvPHP17OkiVr6OubgcXy\nMZdcks4f/nAlmgaXXfY4LS0/ALYA55KYuJaUlCx6erLwes8EvkZv72Ss1gdQHcwUfv+Foc9DN2g5\nFif6QiNMPESsBUEYE4abPBW934IFv+OKKx6hrs5CMHiY/v4errjiEIcPp/L++014vSo5rLfXRXHx\nL5k9+4vs21cPfA7owezG9jFvXjq//e1H9PWpnt2appGVtR5Ng5KS9bS0fAEl1KpEzO/vxu83J5Op\nuHU2Q3U0G6pBiyAMFxFrQRDGhOEmT0Xv19xcQFvbuwQCqrlKe7vGe+9VUl9/RSimq++7Ba/3Lhoa\nLMDVKFFNxyioU6Y0AqezdStE13qvXFkbcp13o4+1VEQnk6k1580LkpS0nrq6AG53K8aOZhJrFk6W\nURfrkpIS0tPTiYuLIyEhgT//+c+jfUpBEMaI4xncEZ08ZZyqZTx2sCSrDz4wj89U4zTBZjsQmtTV\nCfQxUFQvJTPzfk4/fSYdHXvp7k4JZXfrDVKeBRIpKPDQ1FRApGb6d6huZQNbnVqt71Nc7MLh+Ao2\nmxWXy80ttzzP9u1/ALKZNy+Iw/GVkXvIwinJqIu1xWJh/fr1TJ48+dg7C4IwrhnKtR0t4qtWzaG7\n20Ni4lr6+3PIzm7i7bcTaWubA3RTX78E2ExV1VJWrDiDmprb8fnsQCtvvHGE9HSLaXympn1ISclL\n+P3dJCSsIRBIAaZjdkt3A5NJTw8wa1ZPqO3p86HvazDXSa8LvSQsAZ4DJmOxrCEjYzpz5/aQlGRs\nxLLc9EJis1l5/PFvfRaPWziFGHWx1jSNYDA42qcRBCEGGMq1HS3iu3ZV4nTmoeK8GbS3dwA/wxgH\nbmrKZN++JhYufJ5gMNKk5PDhDcTH/4NJk9agaTPw+/fh9f6UhgYbEXd3MlBCxPX9D1QG94N0d/tp\nbEwNradPwUrH7GrPCZVYbWbfvgQ6OtxkZ5/HzJlHcDiWhsW5o8NNRcXwPAmCcDJ8Jpb1jTfeiMVi\nYdmyZXzzm98c7VMKgjBGDFUXHC3iym3dBhgbjAxsKnL11c8RDH4u6rsM+vtn0t9fTmFhJU7nl1FC\nrH/vB3YDS1HWsga8AawGLHg8GocP672+F6Ji1fuARabr1jOxy8s30dCwCqfTEuopHvEWqNptFdeu\nr19CX99fSE5OEvEWRpxRF+uNGzeSm5tLR0cH3/72t5k5cyYXXnjhaJ9WEIQxYKi64GgRV7HlwtBn\nN7AntIKKERcWNrBq1RIuvvgg8CEDZ0C3As/jdPaihHmx4ftEYAZKhDOIWM+6ld1FZmYuUGmYnnUd\ncB9wNoWFDTgc14Xv6WjeAn1spr7+jh1xuN3SHlQYeUZdrHNzcwHIysriyiuvZPfu3UcV69zcjNG+\npDFF7m98M5Hv70Tv7fBhNzfdVMPHH6czY0YXjz66hKwsszX56KNlrFixMbRPN2vXfotLLnmMlhYN\nFS+OuMCnTbuPd965iRUraggGVwGfALejYtDtKCs6H7gUaADsqIEaBSj394LQmkYSME7n6u6+j6lT\nz8XpXBzeIzW1kEWLjvDwwzeZrr+oqMf0olFU1EtubgZOp41ob4DF8qlpm9Np+8z+zkzkv5sw8e/v\nWIyqWPf29hIMBklLS6Onp4c33niDm2+++ajHtLd3jeYljSm5uRlyf+OYiXx/J3Nv5eXPhePRu3Zp\n9PUNZk3G8+tfLzJtOf/8PGpqNgD6HGkACzk5Z9LfH8/evSmh7XagAvgD8AVU/PkHRIu8Emz98/6o\n78wtSW222WRkfAw8hbK+D5OW9iF7987lO9+pNrmvf/zjL/DGG5W4XNOw2Q5w221ltLd3UVjYgdHi\nLyxs4ItftFJTY9zm+kz+zkzkv5twatzfsRhVsT506BA333wzFouF/v5+Fi9ezCWXXDKapxQEYYQY\nbhnWiQ6baG4uQLXhfAyj6L3zzm7OO28PZ51lrImeDEwFFpGfX09Ly2Sik8JU0xPlyk5IsBIIGL/L\nMp1j5swedu7sBH4Y3tbevoH29qvCCXB5eWdjt3fi8/nD7u7eXo21a9dTVWUfxOWvXOdJSdIeVBh5\nRlWsTzvtNKqrq0fzFIIgjBLD7TA2VFLZYOValZVvG9qGHgkd14MxvqxpBTidNxIM3kNZ2XoaG1Np\nb3+Pnh6NuLg/cs45mZx//jq2b+/A7Y4khcFeVCMSK3APA/uFb8Bq9VJcnIDDcTnnnVdLdOKa/nun\n8/M4nUuor9ewWv/IYC8jQ7UClRi1MBpIBzNBEAZluBbzUEllt9zyElu2qGzv+nqNF164g0DgrvDn\nBQvWsWDBOmpqfIbzgJpkZaGz024Yp9kTfnHYts1Fbu6DdHUB3IPFkk1c3Kf09/8EJdQagUAPkYSy\nbiwWK0uWBHA4rgx7B1SSmwvVSjQNleR2KcqKj7QKhUMYhV+6kQljwcBZcoIgCCiLWYkUgMb+/R9S\nXv4MLpfbtJ/NZuW++y7Hbvewb18ql1/+BCUlz7FtWwuqMxiAhUDAjlH8X3stgaSkROLjm4GvEmnr\n+S7gQtP2hs9lfnF4hvb2NPr7LwJmoWnX0N9fBNRgtT5KYWElKhltOSpLfDmTJ3cDsGzZW+F72LSp\njEmTfhnabwnwMzIz/5NJk+5AJao9BbiYNy+DsrL1nHfes5SVrT8h13ZHh5vy8k2Ulm4b9BkKwrEQ\ny1oQhEFxOEro61vHK68ECAS6cLu7qK6eyvbtf+Svf/22KX5tdJmDhtO5EZXBvQG4FiX6jRgt1N7e\nZKqrl2Ox3INxUIYS2Pvwem+jokJ1MTO72l1EN1BRMenFTJv2JJ98YkH913Ynykp2091dSHV1PHBZ\nKCb9MHl5ZzNp0gy83sgLRFzcJLzefwuvXVhYyUMPXXfStdLDDSkIwlCIWAuCMCg2m5Xk5CQCAWPj\nko20ta3hllseISkpNRx/bmxMY2AfbjXVCjaj6qKTgcdR86SnAN9ATbmaiu76jhxfAPyOLVuslJc/\nw+rVc9Bd7Q0NGVHJY16U21qjo6MJj2cFSvwvBHYBd4X214UdnE7V5ASexBzbzjZdR17e2SPS1ORE\nk/AEQUfEWhCEIYkWGV2Et2/vwu3+HrqlOGXKHShhzkANuuhFiV8Lqq92I5oWaRmqLG5QrmY/8L+Y\nG5skAT+jr+9RqqsTqav7G8XF8Tz99Bxuuul5tm0zCmwLaWk9/PM/r6exsQin02ilw8Dr7yISz+4h\nM/NeZs48C7vdg8/Xbyq9Gqn49FBJeIIwXESsBeEURs/YdjptFBZ2DCjPihYZFVceaIEePpyAcRBG\nQsJdBAIbUNnZk5k82YXbbRRNN/BrlKtcubaTk9fg988kGExBNTbRXd7fwe22UF2t3MdJSQA/B+ag\nLOrvEx9fFWoN+gy7dycSEWM9acyGPiHL6+3E6707fK3p6ZVs3apmTbtc7lEpvRoqCU8QhouItSCc\nwkTHmqNjqQ5HCUeOPMJrr2kEAk7i4rK45JKH2Lv3CGoaVTdwMYFAPkbxPuusc5g5s4emptcGtVhV\n1na84RgbfX3TgY+BuahxlQuAHAa6jzNRLvUl4evs6ckIX++WLY/Q16eL8SLgDmy2WcyfH4fDsZxv\nfGMnu3dH1szOLgqvM1Q51skyWusKpw4i1oJwCjBUg5NjxVJtNitPPfUvpm3l5ZtoabkFXXgtltvR\ntHOIxH5d7NnzNh99VITNtodHHilD0+CVV+7E75+NillfS2Lievx+o4A3Ar8wrZuRkYHHE+0+1qiv\nbzadr7+8jtPXAAAgAElEQVT/AKWl27DbO5k58wzee89oxV/A7NkJVFVdBsDMmUdCAzkiDVIEIdYR\nsRaEcc5wOo0NlY18tFjqcAVe07JQFnEVqnd3F8FgJb29quPX0qWVzJ07Db//34kI8wbmz8/kf/7n\nDrzeuSh39udN606ePJudO6/kllseYfv2LiAbn6+fn/98Hps3dxAMRlzdmvYL6uvVveXn/wJz0th7\nfPRRIeXlz+BwlIhLWhiXiFgLwjgnWojr6h6guDjPJNpDWdC6cKmYtcskXMMVeNU0pNLw+bemc7lc\nBQPOHxfnYc8eNxbLLJQrfSHK9R1Zd9IkJwBJSanhZLaaGo2kpPV85Svp1NQsN5wzsnZPTz6RjmgN\nwApcLls45l1VtXTYLunhtlwVhNFGxFoQxjnRQuh2n0l19SKM8eehLGg9ljrYoITIum6ghq1bCZdR\n9fWtY8eOOI4c2Y/ff6bp/Kq1Z+RcmtaI3T47dP5O4EWCwSAtLbOAr6FqoTcCC4iLu51g8MvAEVpa\nfkBFxeZBXzSefnoO77xTidM5DeVWj2SS9/a2ohLXQL0IbEHPAt+3L/64BFjqo4VYQcRaEMY5g2ds\nm+PPJ+L6LShopr7+KVRJViK9vUuorp7MSy+t4Z/+yYbb/R3gEeAAZrezF2OrT6/XRm1tVyi2nQXc\nZth3I3ANKSl9XHbZ0/zP/2Tg8fhQXczcvPiik/nzC03rt7Q0cMstzbhc01D/hX0/tE4asAO/f7ph\n//0YG6h89NEdfPnLTtzunzAcAZb6aCFWELEWhHGOLsR1dQHc7kkol7Kyns1WpMbTT885DjduIsZy\nLF1Yvd6LeP31N1EW60qUtfwEaiBHO6qLsdFFvQGP59rQPtEzoPuA5wgEGnj11UmGLO6rgY34/d+n\noeEOpky5k9bWTCCHlpZp1NT0oCzq7xPp7f0ukAt8E3gUVfaVazqf13s+Xm8iwxVgqY8WYgURa0EY\n5+iubJfLTUVFbbhcyuG4nIqKod24RiEvKurh7rsvNQl5c7O5bEpZyhpwhP5+O5GuY1ZUE5PridRG\n3wecg5o9nQc0AR+iyq6Mk7KSgCX4/Xpf8IENWDyeWSQnt2O2yO8AZgPVqBeEbuAW8vP/MzQ+MxX4\nDip2bbT6+1CW//AEWJLRhFhBxFoQJgjRtbwdHW7q6gIMZUVGx2O3blWJafooy/37WzAL3QcoUfwq\nmnY/0EYkVmxsF2pDCfWi0P7LUeJ9F8oK3xD6eRi4OXSMGo9pPp9qwKJp+4AZmIXc+HKgERdXyeLF\nm1m9+uusXbuerVuht9eC8jJsJDXVj9V6EKdzRegY87jM4T5TQRgrRKwFYYKycmUtbncyRgFsa3sX\nl2vOoCVYbvdsqqt7eeGFZwkEfoBqevI4CQkHiY930dcHyjLdiKaBGtDxBMpSNSd5KYu6m0gnsjwi\nVvi1obUnh36BalGqhFUJ//+GjrkTTZvOpEmfmu7DYslG0yLXnpmZHxbVqio75eXPhLK/rcByFi3a\nyN13XxdOWLPbzeMyBSHWEbEWhAmKEuPLiCR7fYDTuWKISVYaKuY7hUAgDlV+dROwhUDgCwQC/wvM\nAv7VsP8TKAs3A9iHxXIP8fF5BAKTQts04K+AhylTPqa11Xiut4F+LJZ7yMgoJDHxQw4f/hg4HdUi\nVG9peit9fRZaWlwUFlaSl3c2druH7m6LqT/4vHlB071Hu68ffngJ/f3xYiUL4xYRa0GYYOix6P37\nA8ALRMqjGgAL+/bFU16+iX37EigsrKS7Ow+PJxXoR8V6VwHPM3Bs5YOYXdFeIq7opSQn34HFkkIg\ncD1qulYkOe3ccx8hGFxDe7sVSEFlmF+IpnnxeBaQmPhbIuVWoCzvFoyu9by8s009vCsqjLHkr5ie\nQbT7OitrYGmaIIwnRKwFYYKgi3RdXWu4NElZqA8CU1GZ0y/S0dFEQ8Oq8PcLFqwjI8PCn/6Ui7KI\nLaj4cXTCVzbmmHIbcC9wJtCL16u3En0KJeSRY1991UJcXAoqSWwjymqPZJn7/YVRax9BxbUHTwST\nWLJwqiFiLQgThEjC2POYRfZzKMsYrFYv2dlFoVnO6vvm5hxefPEqtmypxOPJRAnkQuBhzHHoD4B7\nUOVQn6Cs9dNQ/43obvR7UWIcB/wB1VAlm2CwjWBwNsYs78j1pQE+EhPvxO+/ECXUXwX+jB7DLixs\nwOG4bljPYbCmJ7m5GcN/kIIQg4hYC8I4xihMjY3NKGs0Oqv6g9C2i0lNbaGpqdXwvYuWlgYuuiie\n1FQfHs8hIsM0koG1wNmo+dR+4N8M696DuQ57PxExVo1U4EbD9/eGfkZf35vArcyf/0fee68Bl6uA\nYPBBEhPzSEhwMW9eBg89dJ0pGexoXcgG6zr27LPXj+RjF4TPHBFrQRjHDBxxuQFlFW/AYulE0yaj\nksImAz/B6ZyDsnZ19/X7tLTcTkuLGiepYtjxeDyRrl9qNvVUVDmWbhF3hn4+jxLfhahY9FMoV/jn\nQvsaLegzQ9fXgYpPzwQ+4owzTufsszfj82XidN4aPm9f30ZgOe+8U3nU+46uH5euY8JEJG6sL0AQ\nhBMnWpiURfssYCEpCVSZlJVIzPkaVLz4Z6i4snnSVV7e2cTFTQltawLuIxA4DTW+8gPUCwGooRv/\nhnKTXwO8SE5OV+j35aiMbo9hfw34O6rL2b+EzvuvQCVnn51OVdXSIZqwdOJ0JvGlL71MefkzuFzu\nQe/bKMh2ux7rVueVrmPCREAsa0EYx+Tnt2N2KX+KsliXk529FqfT+F02A8XQQ3QS1/79h4hY6SsN\nx98R+mVHZY4b1+rC69Vbieq11A+jeocfRjVKuZWMjN/j9f4Wv/8H4WN1oR28x/mLwG243Zbw1Kyf\n/ewC3n//TZStoWq5jYIsXceEiYiItSCMA4aK0VosAZRL+xxUYtZNJCb+ioUL13PTTZdTVnYHXu8Z\nKBHXk8f0lqC7gHxgLSkpUykuDuDz+QkGe1FCrTcyIfTzDOA6VH11E+aXhAz6+oyNS14GvoDKLs9A\nxbxtBAIF5OYewOmcjHLHv0hjo5fzzvt/ZGbmUlhYSUdHPl7vx8BZKE+B2YK++urn8HrvDp970qQ7\ncDi+G35WkikuTERErAVhHDBUjLa5uQBlyR5BWco1zJ49i6qqpZSXb8LrvQtdnJOSKklIuJ2enjhg\nGiqurGqws7PvIzm5kOrqG9HHWMI+zILsDH13AGVdr0G9JAAsJCnpMIHAGjStCJVsdhvKotbLxzR6\nexPp7b2JwsJKenoScbt/gsdjwePRcDo3AuUUFq7F6bwRlQneHzpeXdP+/V48Hv2zcu9bLGdIJzJh\nwiNiLQjjgH374ol0IusKfdZdx06MYyA7OytDx6QSsUq34PPdh893H2bX9qNAKocPF1BX10JEBK8F\nqlCZ4fkogc5CDc643XD8htC+GoFAK5p2t+E7NaVLfc5An1kNVvLyzgagvn7g4I7U1Azi4n5PMHgm\nyoJ/mMREF37/atzugee12Q6OwBMWhNhGxFoQxgEdHU2ozmJKrDo6lCA7HCVs2/Ys3d0RIXc6M7nh\nho20tzehRk0aB20UYnZtu4Dv0NtrobdXL69agcoeTw/9Mo67/H3U8T5SUp6gtBS2bp0V9V1a6Pca\neXlO2toy0NuPFhR4SEpKHSRGrXHwYDvB4C8M2+8jIeE0/P7I2gkJXSQmPoHNdpBNm5aMwBMWhNhG\nxFoQxgHRjUy6u/MpLd2G3d5JWlob3d03YxS3mpofkJFRScQa34PK3Nbjyrqr20qk3MsKTCUh4X7i\n4vz4fBehksOMAtwedbwPTfuE1auXs2tXdUjw9VjyLs48Mxjq5Z3Ntm3Gmux14USwxsZUDh/eS1aW\nnVmz1g8i+oXYbAdMa3/taykSlxZOKUSsBWEcMHPmEXbvjoiVxzOJ+vqrqK/XyMy8n4Edyyx0dWWh\nOoFtQcWoVwE5KDd2GrAas8t6OZBIIHAPSsD/GZXR/RyRCVr5qASzT9AbpHi9LoqLf8mMGbPp6FiD\nxTILm62ZTZuWMWOGHYDS0m2ma2xuzglN7oL4+ATmzp2KwzEfm83Keef9P5Mwx8W9z2OPLeI3v5EM\nb+HURcRaEGIUYwZ4QcERFixYR3NzDvv3f4jbXR7ay0JcXA4DO5Y9h7Ki70fVNH+CyuZuAaaj/ukb\nBb6XSExZjzHXYIyFq45lFtRkrFzD8Vvweu/ivffUfmVl66mq+qHpXqLLsvLzD1FSsh6n8/NAN/X1\nSwA1DWzTpjKKi+/A650LHCEY/Cm/+c1msaSFUxoRa0GIUaIzwBcseCRUB52NcZqWGg+5jr/+tZfu\n7k+Bi1CW8OmoxiMbMVvRG1ATuIwCvxeoNHzuIjLUg9DPPOC7od8/aTg+zbSfPtXLWGbmcJTQ17eO\nHTvigMP8/e+dtLYas8U3huutZ8ywc+aZc0ICrpAuZMKpjoi1IMQI0bXU+/aZrd/t27twu7+HLqiZ\nmfeTnh7gwAE7s2YFuPRSqKkxCq4+0jJ6cEYGkEpy8hr6+opQJVnXAveRklLIxRd3smePO9yCNLJe\nu2Gdr6EsbTvwIcaBH8apXvX1Gjt33oPXO4nu7kwCgRRUh7PJRJLZrEAaBQVt4WcRbYlLFzLhVEfE\nWhDGEKNAt7Xtwem8CbBRX69RWFiJ2fo1dyCLi8vB6VyK07mFhgYbCQnNmEVZH2kZPTiji4yMOI4c\nyUEN2zgHZWmfTmlpAJhMS8vNqCSyDajGJMmobmcuVAw8DeU670MN67gPOJ1Jk97D5TIniLW0nAbc\nYDi/XtJ1DsrVvhyVABeplT6RLmRHG+4hCOMdEWtBGEPMgzjK0OueIZ3u7ngWLPgdzc0F2O0efL5+\namoiohsMtqISwFTcNxBIxizKHwLrAB8JCXeQmmqnt7cVvz+Prq6bQsdGyrL0TmDLlr2FuW3oY6hE\ntXZUDFwvq1qMst5/HfrcH2rCsiHqOj7GPPAjncjMaj9KvFfQ3Pxa+LmcSBeyW255iS1b1JSv+noN\nn28djz++7LjWEIRYRcRaEMaQgYM4VN0zWPB4FvHOO5U888ylVFa+zYEDqRQWVpKdXcTMmT3s2NGD\nx6N3KFPlUCrTuwg4hEokawb6uPLKqTz++DJKS7dRX38VqtXnFNO5LZaZVFS8SkGBL6r+OQnV4/t7\nqKYo0Znnt6EEWo9xL0QJsB/1wvBjIrHpDSi3ezfqBaAGZWWfvKtbxcONYQOZUyRMHESsBWEM0F22\n+/cHUMlaKlksISGDQCAiOE7n5/n615/D6VyFXtvc3d3K4cOddHbOwFwjnQccxOxyvg/4N/7+919Q\nWrqNtrY9QDHKlW22xHt7J1Fd/VVycx8gLq6SYPBclKguBJ4hMfGXTJ6sceiQUcg7iMTBdXe7FWWx\nbwQuQAm1up+MjB4uuSSN5uYUCgr+Avhpbn52hMqx9AEk+rUdPsn1BCF2ELEWhDEgeg611foAxcVT\n8PniTK5u2EN7+ySU8H0K3IbHsxGP5ybMMWA97mu2llUi1ye0tEyipUUD4oiLewzoIRj8FuamKWnA\nb2lvn40S/UuIWMQp+P134fGsJGJFd6GsZz0uvjD0nQc129ofWidyPxkZbTz00HWjEkueNy+dmprI\ntc2blz7i5xCEsULEWhDGgGj39/TpZ1BVdQUul5va2kiNMXyf/v77gVtRcd9OlGgbY8CHgZ8Dp6Hi\nwy4iItsM/BfG0q1g8FGUqNeiXNyXoBLMsgFzJzRlraeg11/7fJ9HxbHdKBd2H6rZyixUL3Er8+f3\n8NFHHQZvQCRJzelcQUXF6NRMP/TQYpKSamlq6sduD+BwLBrxcwjCWCFiLQhjwGClSR0dbm699QW8\n3lTgXVQf799hsVhD+3Whz3c210w7iSR96f29P48S4FuBNxgqLq72vxPlEg9gdqufTWLiLvx+Y1xc\nb1eqZ3FvBCJWfn7+PVRV/V+WLXsr1B5VT1J7At3CHq2aaRmNKUxkRKwF4SQYrFxI0zhmCdGqVXPY\ntasSl2saNtsBVq8uY+XKWmpqMlGCGBHI/v5VKKH7J1Ss2Si8XSir1rgtK7R/AcrCPow5lpsVtX8i\ng7ce3cP8+Vbee68y1GnsCPA14uJuJxiczWA13IcO5bFs2VuG2Lhu4SeG1tyA3R44mUcuCKckItaC\ncBJEdxnbseNOLJZEWlq+SHQbTSOVlW+H3MRq2tXatetDFmc8oAshqCztIpYsWU9dXStudwCz8LpQ\n7m/jtkmoGHRC6LNuMetx5vej9j8Ns3gfAdYQF5cNTGLTpq+wdu3bNDVl8v77/43X+wsi5VnmGu5A\noIv6+u8BZRQWqpeR3t5EdDe61erF4bhyJB69IJxSiFgLwkkQHXtubc3E7KbeyL598dxww5Ns394F\nZDNvXj8HD9pMx+lWeH19AsqtHRHA5OSPqaqqoKTkJdzuuURiyZ+gBnTEoVzZXyAu7m0SE0/Dau0h\nP9/PO++sQpVyAVyKcksfQrnN1QuFwije7cDdBIMWtm1TLxL6y4YqrzKWZx1Cud3PRDVJ0T0IFvLy\nzmbu3E6qqyO13MXFCdKoRBBOABFrQTgJomPPaqqV0UpN49Chf9DQMBNVp2yhpkajsHAtRoFsa3uX\nRx5ZwvPPr6e/H2ANMAP4kOeeUz2yOzo+QM2n/hmq3KsIVaOs1igsrKS2dkVYDE8/3YHRnR5xbx9C\nJY3prURdpKTcyec+d0FoSMh0ol8kdDIzP6S39ymUlR4EWiksTCMvz0Jb236czhWhPbVQOdbxdyIT\nBGEgItaCcBI4HCXs2mWM6YJRhAsLG+juzid6KEZW1nSCwXtCrTgP4XTm8POf/5WMjM/hdn8ntJ+b\nxMTf8KMfHaCx8QX6+rJRTU+CqCEdlqg1i/jRj14KNQc5hNc7jYHu7TtQGdr5JCSsITGxCJvtIK+/\nfiOZmVmUl3dSXR003YOxWcm5506ltdU8lzovL4etW6/A5ZpDRcVmkzBL0pcgjAwi1oJwEthsVmpr\nr6OiQh9l6QbWceCAlY6OvWRl2Wlvfx9lyUYEsKOjiba2eIwNTF555U5SUuyoEqgkQMPvn857730F\n+CbKMs4Grg8d86RpzY8+eoeGBqMlvQqze/sQytJ+ArievLxK6uuVkObmZtDe3oXDUUJX1yb++te1\n9PfnkJfXyurVXw/fb0tLtOcgKyzmgwmz9OsWhJFBxFo45TlZQRlMpMrLN9HQsCpUvuQCfoXqo51D\ncvJenM6fAq9hFD6//zz8/q8DT2F0b0cGX6SjMrvNk6+s1qmkprbgdJ6FWUhPJ9L0pBs1IUvPFrdw\n6JCV8877T7KzizjrLB93330pNpuVjAwrfv8PUUM4VMz6vvsms3JlLR988AnGF4BJk/6Ow/HdIZ9N\ndAIerBdLWxBOABFr4ZTnZAVlMLGPTjxLTEwmISEPm+0AaWmz+fBDGwOzsveimo34MIuuPviiG5X8\npR8zGYsFtmy5iNLSF4nUQOvrdaJGUBpFXwM+ADz4fB04nbfjdFrYvVtj69YHKC7OGzCas6kp0/CM\nzE1OZs8+86gvNtHPQeZSC8KJIWItnPKcrKAMJvYFBUeor9cTsRrw+1fj96syrbi421GiOR01ZcuF\nSkzTUIMyEjGKrsXyDzTtDSAXaMNYhlVSMpnKyrfxeH6KEtInAC8WSxslJalYLI/wt78l0Nv7CX7/\n5NCx/4pqQ/p703273WdSXb1owGhOu91jeEZ6k5PNwCJmzVp/1Gcjc6kFYWQQsRZOeU5WUAYT+4IC\nH2ZXduT7YLAIZeUeRDUNKUSJbyJKjL9NxH39Ppr2A5S4bgD+D/AUcXFTyM9vZu3aMr73vY9QQl2D\ncnHXk5CQQnp6DqtWzaGy8m2ami6goaGVQOBaw5V7MFvi3YCFtjYbmZn3Ehc3hXnzgjgcX6Gi4lXT\nM7Ja36e42HXM7G7JBheEkUHEWjjlOVlBGUzsm5qMiVjdmEVxHzAXZSk3oTK0dSt6DZo2GX1spGpu\nYkW5x53AfwM/Ixi04HSqeLLdrlFf/yKq8cgW4Iv4/X+jujqVHTueprVVTzp7LOo6JqFqtnWL/VpU\nYxM3Hk8u8G2SktZjs1kHeUbLhxXXl2xwQRgZRKyFU56TFZTBxN5siS5g0iR9OEcD5vnOD2O0ujVt\nKuaksNND3+k9wfVhHjVAOi+++AlPPTWX6upGlFDrDUgWAxtob3cb1r8KPclNZZtnYh7c8SAwFfg+\najZ2JCQgoisIY8uoi/Xrr7/O2rVr0TSNq6++mu9+d+jMUUGINYzJY0VFPeGM6ejv7HaNp5+eg6ZB\nRUUtH3zQR1LSSgKByWhaFgkJCeTkvMGhQzMwzneO7tsdH3+Q/v7lKOFNA/4GrEe5rI3DPJSL3e9f\nxHXX3YHqIKYnkaWH9rMQF5dNMOgCnkPVWbtRZWT7UF3QjIlsn0OJPOgxdIkxC0JsMKpiHQwGufvu\nu3nsscfIy8vjG9/4BldccQWzZs0azdMKwogRnTzW1xfJFDd/52LXrofp6cnH7U5GtQA9D11Uu7s1\nurvvZaBLvBdju87MzG5crl+i3OTdKGv6d8TFHSYYfCp0nC7cABb6+magyrgexNyx7A4CgWyUq/si\nlBv9bsP390ZdS1doTY3MzGYuv3y9xJgFIUYYVbH+xz/+gd1uZ+rUqQB87WtfY9u2bSLWwrjhaJni\n5u+2hAdzRFzKUzBbrlNR85/10qckoAKYTGbm/Vx+eT61tfmodqLGcqtzCAZ3EElYM2drJyU10tc3\nGbgg6nznh873o9Dn56K+n0JcXCWZmflcfHEQTfPT3PxsyJX/LWleIggxxKiKdWtrKwUFBeHPU6ZM\nYffu3aN5SkE4LvQZ0sYhGw899NWwUB0tU9z8XRpmIcxhYLb1p6h48JbQfsbM7CyqqpYye/bTUesk\nomZbzyYya/paVO/wmUAj55+vMWXKeurqWnC7jefrwzzCMtqqn0QwuBq3WyM9fSO//vWiE3+QgiCM\nKqMq1pqmjebygnDSRGZIR4ZsJCVFXN3G5LGiol7uvjviFjZ+19a2B6fzUpT1qgGfEB9/iJSU/Xi9\nOaSmdjB3bhJJSX/hwAErDQ31GIWzp6cJgNTUZjweo6C+jZqQFT2M42z07O3333+A555bSmNjE5dd\npiey7UG9GNQYzrMAWE1CwukEgx0Egz8I3YmFl1/uw+VyizUtCDHKqIp1fn4+Tqcz/Lm1tZW8vLyj\nHpObmzGalzTmyP3FFk6nMdlL/Xz5Zbj55s08/PBCiopO49lnrx/02I8//pitWz/C651OUpKb5OT7\n6euLCGt//4P097t5//2vMmuWPXzcsmUbaGiwY8z61rQscnMzyM+fTUuLMRu8ELOl3YNyg98U3max\n5JKbm8HNN+8OCfUSYD5KqA9jjImXlc3i2Wf/lWXLnuJPf5ocWkPD5UpizZo3ePrpa07mccY04+3v\n5vEi9zexGVWxPvfcc/nkk0/49NNPyc3N5YUXXuCXv/zlUY9pb+866vfjGX1YwkTls7y/kRoQUVjY\ngbI8jVauxp/+dA11dXdywQWn09ycg93eyaOPltHfHx8+trj4v/F6VUJXX9/AMiz4HL29izjnnDWc\nddaF4evcuzcF1RAl0go0Le1+PvjgAG1tjcBqIpb0PZhd1wdRLvGI0H75ywHa27tC6+qubiuwHKv1\nAdzuVeFrfuGFRzj33Cc57bROMjPvx+M5K3TMQvbufW3C/v2Uf3vjm1Ph/o7FqIp1fHw8a9as4Tvf\n+Q6apvGNb3xDksuEESE6S9vne4SkpNTjFm+Ho4QdO35Pa2ukhSf4AQutrZnU1NwYPseKFea4rsrC\nNoqzLvzmjmB9fRdRX78k3IpUNTHRO5KloHqEZ1BS8gRO57+gLO40EhPfRNM6CQSM1xYAesOJYfPm\nBXnooa8Aegx9Sfj4KVPexGJJQLnmu4EFBAIZNDRYaGhYQWHhWjyeReHrLShoobx8k0zIEoQYZNTr\nrOfPn8/8+fNH+zTCKUZ0lvb27V243SrufDzDOGw2K7m5X6S19RuGrZtRYmseB/nxx+mmYxMT38Pn\n08up9gNWLJZVaNoUVCb4wtA6R8JrNDVl8vTTc/D5nmf79k85csSH378aj8cSilXrE7bgnHOCfPCB\nJ6pF6BPAdSxePPD+VAxdnyftxuc7Pfyyoa7j58A09KSzrKzpzJ0bicd3dSXIhCxBiFGkg5kwLonO\n0lZzno8+jGMo13lHxweYLeJ/oCxRv2n71KkdpvXmzTuNurprUAKryq1UUuUToWP+GlrrJlQzkhfZ\nv99LRcWr3HnnpVRWvs3WreD361neVlRWOeiZ521tB+jtjVxDYuJHLFwYqX8+WjigtHQbZsv/QmAR\nqu5aY9as/rAY5+ZmcP75zx7zGQqCMDaIWAvjkugWnz5fPzU1Rx/GMdQozKys6TidxqQuG5BGUtLf\n8fkiLmhN8wMRgfzb3zKJuLKNomhDJXlp5OfXc/75f2H7dhdu909wuy1UV2vs2lUZVZetZ3nvITPz\nAOnpnTQ2FnHWWekEg/fQ2WnHZjvIpk3fZMaMSLLa0cZ75ucbx2lG3PLJyZlkZ1fS2FhEefkzOBwl\n5OZmyIQsQYhhRKyFcUl0r2qXy01SkhLv/PxD+Hx+Skqeo6OjiezsImbOPGKY0+wGati6FcrLn+G0\n047Q0BA993k+gUAzxlpop3MzYBbIwTuBvQtYsFrfp67u/2KzWSkt3UZ9fUTQW1ryMQu8L3TeFfT2\n/gaPZzVOp1qvrGxod/TRmrZYLAHMDViUWz47243TuSo8xxrW8+yz18uELEGIYUSshQmBUbzLyzdR\nXX0jSvwiohSZ01wDLKe3V1m5Cxaso6xsPXV1AdzuScDngQcJBmcCT6JaeU5mxoxuYKBAKsv7DlQ8\nuAOYDnhITvawbNlb2O2dFBT4TFZrMNiIWeCdwCpAw++fynDd0UezhpubC1DDO9TLSUrKc5SWQmNj\nUagP95kAAB2VSURBVOhFwLy+DOsQhNhFxFqYcETE1Ni9y0J2dhFz565n61bo7Y1s37LFD8SRlLSP\nSy9NYceO9/H7jT221wCn8cYbbXz8cdMQ8fJ/Ae4HvoxyN/8Tra1NtLbGU1+vYbM1oAR9BtCIGk/5\nKOACckhI8HHmmU9y8KATtzsXo5C3tb2Ly6WGhETHp49mDUeuU5VxlZYqC728/JmQRS3ubkEYL4hY\nCxOOiEh1YU4QcwNJJCe3mJK21Pzoa+nr0/jb39bQ3z8Ts+V8EbAEp1Nj6dJKamuvo69vHVu39hMM\ndqFqnp/D3GnsTuDfw59drmYiPb9dwC9DP28DLAQCGtOmraOjw4/bnYkS9rMAC07nCioqlAteud87\nqa9/kc2bXyQ//xCbNpWZ4tg6Qwm5uLsFYfwhYi1MOHQx2rcvno6OylDMugefzx9yj3cCG7Bavbjd\nzcC3ULHddPr6JqHKsIyWc6T0yuWahs1mJTk5iWBQj1u7gD9hFvjZUZ+Nru0tqOlYz5v22bEjLtTA\nxAIsxVjGFXGFW1Bu/GsIBi3hF4j6+h8OeA5DubXF3S0I44+4sb4AQTgROjrclJdvorR0G+Xlz+By\nucPf6WL05z/PZ+7cacTHJwAaBw7o7nErcC3Tp2eRnNwN/A8qE/tS1HCM6cDtKOt3FZAMPAW4sNkO\nAkZXuxvVuSwdJewQGdox1Gd96EdX1D6HMQu8uYzLbu8M7Wd277tc007gCQqCMJ4Qy1oYM06mZejR\nSpaG2ieSYBaJ1WZm5vL66z6MFmvEoq4M/VKfU1LuZNOmbwJGV3sNKiFtPpFe3/XAdeidxPLz/8E5\n56Tw1lsPEAza6OvbT1/fYlR2trLwi4sT8PnSTOVn8CbgJjHxQ1avXoamESr5ysKY+Ka/QAiCMHER\nsRbGjOEI7lAcrWRpqH2ys4v4whfWsWNHHHAYny8Nl+t0Iv20zRYrmMurZs/+AmvXvk1T00cUFPhY\nsOB3vPZaGr293ai49TWhdeopK3s93EnM4bjB9BLicrmpqNBjxgEcjiux2azh8jOVld4K3ArY8Ps1\n1q5dD2CqzY6LqyQ/HzZtWjKsZyYIwvhFxFoYM4YjuDC4BR6dkd3W9i6NjbOprHw7FKtuort7CkYL\n9PDhvRw4kIjb/RNAjcMsLFwL5KFi1p+iOnzplu1HGC3xDz98h927fwxsob5+Cvn573DxxbBt23J0\nK1qNpnRTVXXLkPetu+n1+9LLuxyOEqqqluJyufnSl17G7Y5MBDPHrNXPL3zhbLZuveL4HrogCOMS\nEWthzBhux6zBLHCHoyTkEv48cASncwVf//rDIctT1Vfr61qtD5Ca6sfpXAG8gVHwsrKm09PTh9t9\nLSr+vBHoBeJJT7fS3R3pbOb1TkElhy0HOmlp6aalxQU4UAlk76EakMwIdwY7mls/+r5eeukOzjjj\ni8yceYR587yDdGTTpMOYIJyiiFgLY8ZwSog6OtzU1bWiMqe7gIXU1QUAyMs7G6cz4gJWiVadKAs5\nsj9kk5WVHJpdbSznctHR0YT6Z2BM9Ipj0qQP+dKXckNWs3EQhje0dgORUiy9i9mtqBj2tVRXR9z6\nQ8Xmoz0LXu9cdu9ewu7dkUYtA5+NlFwJwqmIiLUwZgynhGjlytqw21qJ4gbc7klUVNSGRk1GLE2b\n7QC9vS+i1y4b909N3R/6HEnqSk1tCVnincCDoTOqY71eDYvlEcrK9CYqicBpgHGKlTG+fQ7K6s4I\nb9Nd10PF5gc2V4mUiG3fHsfOnZcPsMyl5EoQTk2kdEuIaQa29vQBC2lqysThKKGsbD3nnfcsZWXr\n2bSpDKvVO+j+2dlFoX1fo6wswM6dV5KXdzaRUq5CoMh07JtvJlFVtZTSUg3l+p5i+F5PSoOI0Kah\nLHe1TXUecw8am+/ocOPz9ZCYeCeqocq9wFfDx+ovJIIgCCCWtRCj6K7j/ftbMDcoSQYmY7d7BrXM\ni4vfCrmgzfvPnNkzYF+zZbsA1S50cfjYI0eacbnchvi4RiQBbQEqLn4xSqi/Sn7+b9C0Plpbn0OP\no1dUbB7gAbDbPaxcWUtNzfdRVv2LZGZm0Nv7K/z+81Gu9oU0Nb02sg9VEIRxi4i1EJNEXMeq21hm\nppf09BaysuzMmrWeVasuoLx804A4sB4Hb2xM5fDhvUPuv2LFGezc+Qlxcb9H09rQtGyU5RwZien3\n9/GlL71McXE8mzYtYfHi/6Kt7V5UMtmnXHppOllZ7tCam3E4bmDZsrdobdXj6CrePm1aIYWFkU5q\nDsflLFv2FsYGLTNnPovdnkF19VVE9wQfbu25IAgTFxFrIWYwJmIpi7oTo5ht3fp/wvuqyVqROPCO\nHXdywQX/v717D66yvvM4/s4dSAI5QIBEuiGAEay2TC11YVxCsY0SwKBopXWkRZuV0sEx7Qw3124t\n3VBTrbZDhyJip1AqWNYkUAhVA4RWKcvWTTEqZYg0CLmS5DQJhlzI2T8eTs41yUlyDufJyef1jyR5\n8jy/x4if/G7f379QVTWelBQb+/bdicVyj5frjbra+/f/DZttKvZtXUbFskSMFd23AGeBWVitVyks\nXAgcYO7cmRQUrMAepnFxO/rorR/qPsMabMye7VhwVlv7IcYsVAuw8PqCMc8V7mvXHtA8tYgorMU8\nPM+Jfg3jPGnPbUru88A1NaMpKjIWf5WW2igpeZ709AleVl4bVcpsNvszXgVGYdTyjsGo8x2O8yEc\nsIeKitFERUW4PLOqarzHO2zYcAenTm2msXEyHR2tdHZ67iNft+6oS3GT5OTN5OU9isWS4LHCvbfj\nMUVk+NACMzEN9wBOSLjavXjMfZuSo0421/853uV7rdYZFBau6F6k1VNdbSOclwMP4Dhw4zxGr95+\nTSy1tR9y7twZl2c6/wJhr1V+773/Q2VlCq2t99HZGef1evf3nDDh1u6hbvf30l5qEQH1rMVE3Lcy\n/eu/dhET00RFxWjWrj3iUmTEfcjY4LywrAXn3qx9LrukpBqr1blKWTze64I7evUjRpyisvJx4G3g\nBSIj4/nqVyPIy3MMs3uOCuwBFpGQ8DyTJ6fS0HCW8vIUsrPfICmpvcfiJjq+UkS8UViLaTiOthxF\nQ8NZ3n13NE1NI4H5lJaOwbl2uMWSwNGjj/LUUwc5caKZrq6RjBr1X3z6adL178nEOQjtK8dd63I3\n0dLSRXGxZ4979OirhIe/CtTT1TWRq1dPYN9j3dlp429/20xj4z9Zu9Y+x96Ja489DhhDevpE4FPK\nyjZQWRlGWZmNhQt/1UPBEx1fKSLeKazFNOxBlZ2dT1mZY07Xfq6z+/ytxZJAdPQorNYngDCamowg\njI6OoqLimNeeqc3m8hG5uf9Gbu4ujh6toqnJ0eP+9NOP6ezcdP3j3TiOtQQIo7LyNpYuzae6ehoQ\nAdTg3LNPSDhDenqj28pv43urqpJU01tE+kVhLUHR2/GYnoVQjLlfb/O37td6C0LnZ9XWfkBl5SPA\nCUpLLZw6Vcirr36ZoqIyjCpmxtx3Z2eM030XERb2PDabYw82NFJdzfW2NQNfJyrqP/nsZ79w/ZeE\n5S7z0KrpLSKDobCWoOjteEz3cHPupdo5iqZ0Ar/AmKOezJkzZzl/fjqpqSlenwVZwHPAOowe8hKW\nLv0B7e3P4dqTHwf8DmNOu4nY2Gu0tPwEo6zoFYzKaP/h8j2xsVO89pg1Dy0ig6WwlqDo7XhMz3Bb\n7lIYpKHByoIFu64vLmsB/oF9q9XVqzbuv38zpaVrenyWUVrU8XFbW6rb12MxVoR/B8ee6k20tKzC\nqP8dS2Rkk8u2LIhlzhz7QjdXmocWkcFSWEvA+XIetfPQcF/h5r5PGTbjHLaNjZNdnlldfRqoAyYB\nTURHv097u+PZMTEfc/Wq4+Pw8L8QE3Mzra2OeyYm3sq8eYc5e3YkKSlW2tvDXY6wTE4u46WXHvXr\nvyNVLhMRO4W1BFxP51E79557Kh/qjWtP+Z/ANYzDMIxqYBbLRS9D369h1P22MW9eM7Gxjmd/97uZ\nfOtbRiETi+Ui+fnfIDfXtcb41KmfsnfvCurqjIM6GhutREc79/4fHVS49jYtICKisJaA8zbk7d57\ndi8f2ltYTZpUh2Pl9SGc545HjPgB+fkP88QT53Ad2o4HrEAR77wziowMG3v3Oupul5be7vKMvDxj\nq1hP88z+HtrubVpAREQVzCTgfKnK5R5Wb74J2dlv0Nho7b7GXiXs3XerMHrKBzAWejm+b8aMO0hN\nTfFS4awZo/DJclpbV7hUN3O/f0ZG8fUiLF9mz547AFi27CSf+cxmFizY79Euf1DlMhHpjXrWEnB9\nrYb2drBFa2sUhYXLgV0899yXWbfuKCUlnVitMcDNGNXGwFix7Riurq39kIwMSEq6wsKFO6iqGk9S\n0mWgg2PHYl3mod17r84nfZWWHqKk5C1Gjap2mR+/eHEPZWUr8PcwtVaMi0hvFNYyIN4WRCUmxvf6\n9Z7mdD0XjD0HrMIeqJ6lPH+CUdP7MAAjRjzDzTfPor7+LJWV36Gy0kJpqY2srF28+ebd3W2Jiemk\ntXU39pO2ej4cxCg9arWGYbXux3PPt/+HqbViXER6o7CWAfG2IMo4PrLnr/cURp5bq27FOBrTGA72\n/PoM4JcYx1oa27UmT95BRMStVFZauq9zPuXKOewTEp4nPX2i18NBjLY6lx5twbPmuIapReTGUljL\ngHhbEFVfbyU7e7+X86h774m6b+OaNOk0V69eBuppb48lKanNrUjKOVpaEl32OZ84EU56uvftYO5t\nnTLlZrZv77l4iethHwtJTt7MuHFpNDaeIyHhM0yb5jgFTFuuRORGUFjLgHjbJ716dZHP51E7y8tb\nQFvbDv7yl3CgHputDat1JRBGUZG3gy+Wc+edr2G1Ovd468nLM+a43ed9fS332dNhH/ZtWYmJD3Zv\n3bLTlisRuREU1jIg3vZJL1z4v7ifRz1lSkGfC6YslgRiYqKxWpdgzEN3YAR9JpDgtd73nDlxFBW9\nhrElq5k5c+KwWBJYv/4LLFu2n7//PYk//nEbqak3M2WKY7GZL4u3+jN/rC1XInIjKKxlQLztk25s\njMJ5fjc9PdLrcLOd8xCyMWy+H1iBo7e8B1jutSf80ktLiI4+SkXFNVJSOsnLWwzAsmX7XRarffTR\nHj76aEX3YrPBcB7m96USm4iIvyisxS+MHuV8jIAdQVTU/1FefgvZ2W/0OI9rDCHbe9MzgCqce6kj\nR3aQkbGrx+pm3nq/jY2TXe7hz9XbzsP8PVVi05YrEQkEhbX4hdHDHIOx//l3dHQ8S1lZGGVlrvO4\nnr3p/wYex3FutKOXmpFB9/nWvs4LWyyf0NoamNXb5887rxL3XolNRCQQFNbiF3l5CwgL28mxY9do\nauqgq8v7PK7nnukXcD43Oioql8jIz2CxXGTjxvsAKC8fhXNIfvzxKI/n238JGDNmOg0Nz2CzJREW\nVk1q6nTS0nb5pcebmtrMqVMa8haRG09hLT5raLDy1FN/vL5q+zJz5sTx0ktLsFgSsFgSiI6Oxmpd\njrE4zHuouS/IioyMp7PTfu0YOjpS6ej4Bq2tNnJzd7F9ewoNDX93uV99/VngHpe2uf4S8DWysnax\nffsK/Gnr1kza2jTkLSI3nsJafLZu3VEOH7YPWdsoKnqN6Oij3cPAjmHiTGDP9TlnXELNfUHWqFEN\nxMUZ+5g/+eQ8Vms29gM39u/v5NSpXxAbm4QxFx4HtDB2bIpH227EquyxYzXkLSLBobAWn3lWEoun\nouJa99cdw8QJwHIyMjznlh2FRzqxWkfQ1PQdmprGMHv2LqZOnUBh4Rjsq8BttjAqK22MGPEMsAl7\nwE+btsujbVqVLSKhTGEtPjMC0V6TOxb4gKQkxypvX4aJ7QuyMjKKKS1d2v35iorR7N17B7CLwsJm\nHD3pZq5ds7gVRfG8r1Zli0goU1iLz/LyFnDy5C+prjZqcsMSYEf31/szTOzeE66t/ZCHH4aUFBsx\nMVW0ta3u/lpExA/Yvv3fe72fVmWLSChTWIvPLJYEJk26jepqx1B4VdX4Ad3LuSdcW/uhy2lZ8fHb\naWtzPCM19TZ/NF9EZMhSWIvPvJ07PdC5YeeecEYGLqdlRURYcV79nZbWNui2i4gMZQpr8Zn7udPJ\nyZvJy3t00Pd1HxKfMyee6GjNP4uI2CmsxWfuq8EnTLgViyWhuyBJZaWF5OSGfh8T6bk4bLGOmRQR\ncaKwHqYGcg5zT9ujPKuS9e+YyIEsDtM50iIynCish6mBnMPc0/aoG3VMpHNA19Z+QGXlasCic6RF\nJOQprIcJ957oxx/H0t+A7akH3FOP29+9X9cefBbGXuyv+9x+EZGhSmE9TLj3pJOTc+mpfndf3EN4\n40ajmIkxZ93Y3eMeSO+9N54V1GKv/1kVy0QktAUsrLds2cLrr7/OuHHjAMjJyWHevHmBepz0wT3o\nxo6dwuzZA1tx3VMIJybGU1fX3OMzB9v7de/BJyeXMWFCl1aMi0jIC2jPeuXKlaxcuTKQjxAfuQfd\ntGnXBtzL9TWE+1uvu69hc88580e1qExEhoWAhrXNZgvk7aUf/Fk729cQ7u8z+xo2V0lRERmuAhrW\nu3fvprCwkNtuu43169cTHx8fyMdJL/wZdL6GcH+feaNWlYuIDDVhtkF0f1euXMnly5c9Pp+Tk8Os\nWbOwWCyEhYXx4osvUldXR25u7qAaKwNXX29l9eoizp+PIzW1ma1bMxk71lxDyA8//Dtef91Y3Q02\nvva1Pezd+/VgN0tEJOgGFda+unTpEqtWreLAgQN9Xuu8QCnUuC/AupGys/NdCpdkZfl/X/Jg36+x\n0cratUddeuxmmpMO5s8v0EL53UDvN9QNh/frS8CGwevq6khMTATgrbfeIi0tLVCPEh84hpitQBFv\nvgnZ2W+YqvKX5qRFRLwLWFj/9Kc/5aOPPiI8PJybbrqJH/3oR4F6lPjAsSisCFhOa2sYhYUD3/vs\nbeW2L78diohI/wUsrPPy8gJ1axmADRvu4NSpzVRVTcJmG/wiLm8rtwsKVviruSIi4kQVzIaJzZvf\nu3685Wv0VrnM3mMuL4+goaGCcePSmDr1isdwuVZui4jcOArrYcIRrpnAHkaO7CAjA49tV44e8x5g\nA5WVYbz/vudwube91vX1VrKz9+skLBERP1NYDxOOcE0AlpOR4Qhf5/nnf/yjEyOA43DuOZeXjyI7\nO9+jHrh95faGDV9g1qxfcfHiOvpbC1zHXYqI9E5hPUz0VsjE9TSr3RjD5M04D5dfvnyGsrKnsQdx\ne/sOfvObh7vvkZ2dz8WLtzKQoXF/H/ghIhJqFNbDRG/bolznnxeRkPA8kycn09Cw+fqc9accPZqA\ncxCfOBHu5R4tDOQkL81/i4j0TmEtbvPPY0hPn8j27fe5XJOWthXnIIZ6L/e4D2OuO5bk5DLy8h4d\nwPN13KWIiDuFtfhU63vOnDiKil4D4oFm5syJ87hHTMxhzp4dSUqKtV8nYvnzkBERkVB0Q8qN9keo\nl5Qbqu/nSynQofx+vgjl9wvldwO931A3HN6vL+pZB0ioVfhSKVARkeBRWAeIKnyJiIi/hPd9iQyE\nVjiLiIi/KKwDJCXlnxirpkErnEVEZDA0DB4gWuGsymQiIv6isA6Q/i7ICsVgU2UyERH/UFibRCgG\nm+btRUT8Q3PWJhGKwaZ5exER/1DP2iRCseSm5u1FRPxDYW0SoRhsKqQiIuIfCmuTULCJiEhPNGct\nIiJicgprERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgpr\nERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NY\ni4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NYi4iImJzC\nWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicoMK68OHD7N48WJmzpzJBx984PK1\nbdu2kZGRwcKFC/nzn/88qEaKiIgMZ4MK67S0NLZs2cLs2bNdPl9eXk5RURGHDh1i+/btPPvss9hs\ntkE1VEREZLgaVFhPnTqVKVOmeARxcXExmZmZREZGMnnyZFJSUjh9+vSgGioiIjJcBWTOuqamhqSk\npO6PJ06cSE1NTSAeJSIiEvIi+7pg5cqVXL582ePzOTk5LFiwwOv3eBvyDgsLG0DzREREpM+w/vWv\nf93vm06aNImqqqruj6urq5kwYYJP35uYGN/v5w0ler+hLZTfL5TfDfR+Q12ov19f/DYM7tybXrBg\nAYcOHaK9vZ1PPvmECxcu8LnPfc5fjxIRERlWwmyDWKb99ttvs2nTJhobGxk9ejQzZszglVdeAYyt\nW/v27SMyMpKnn36au+66y2+NFhERGU4GFdYiIiISeKpgJiIiYnIKaxEREZNTWIuIiJicacN6x44d\nzJgxA6vVGuym+NXPf/5z7rvvPpYuXcrjjz9OXV1dsJvkV3l5eSxcuJCsrCzWrFlDS0tLsJvkN73V\nwh/Kjh8/zr333ss999zDyy+/HOzm+NXGjRuZO3cuS5YsCXZTAqK6upoVK1aQmZnJkiVL2LlzZ7Cb\n5Dft7e089NBDLF26lCVLlrBly5ZgNykgurq6uP/++1m1alWv15kyrKurq3n33XdJTk4OdlP87tvf\n/jb79++noKCA+fPnh9x/gHfddRcHDx6ksLCQlJQUtm3bFuwm+U1PtfCHsq6uLjZt2sSOHTv4wx/+\nwMGDBykvLw92s/zmgQceYMeOHcFuRsBERESwYcMGDh06xJ49e9i9e3fI/Pyio6PZuXMnBQUFFBQU\ncPz48ZAsW71z506mTZvW53WmDOvc3FzWrl0b7GYERGxsbPefW1tbCQ835Y9gwObOndv9TrNmzaK6\nujrILfKfnmrhD2WnT58mJSWFm266iaioKBYtWkRxcXGwm+U3X/ziFxk9enSwmxEwiYmJzJw5EzD+\n3zJt2jRqa2uD3Cr/GTlyJGD0sjs7O4PcGv+rrq6mpKSEhx56qM9r+6xgdqMdOXKEpKQkbrnllmA3\nJWBefPFFCgsLiY+PD6lhK3f79u1j0aJFwW6G9MJbHf/3338/iC2Sgbp48SJnzpwJqQJUXV1dPPDA\nA1y4cIFHHnkkpN4NHB3T5ubmPq8NSlj3VG/8qaeeYtu2bbz66qvdnxuKvZi+6qnn5OSQk5PDyy+/\nzG9/+1vWrFkThFYOnC/14rdu3UpUVNSQmyscSC38oWwo/v0ST1euXOHJJ59k48aNLqN3Q114eDgF\nBQW0tLSwevVqzp07x/Tp04PdLL84duwY48ePZ+bMmZw8ebLP64MS1j3VGz979iyXLl0iKysLm81G\nTU0Ny5Yt4/e//z3jxo27wa0cOF/rqS9evJgnnnhiyIV1X++Xn59PSUnJkBw1GEgt/KFs0qRJVFZW\ndn9cU1Pjcx1/MYfOzk6efPJJsrKy+MpXvhLs5gREXFwcX/rSl/jTn/4UMmH93nvvceTIEUpKSmhr\na+PKlSusXbuWvLw8r9ebasI0LS2Nd955h+LiYo4cOcLEiRPJz88fUkHdl4qKiu4/FxcXM3Xq1CC2\nxv+OHz/OK6+8wtatW4mOjg52cwImVHqkt99+OxcuXODSpUu0t7dz8OBB7r777mA3y69C5WfVk40b\nNzJ9+nS++c1vBrspftXQ0NA9PHz16lVOnDgRUv+//N73vsexY8coLi7mZz/7GXfeeWePQQ0mnLN2\nFhYWFnJ/0V544QXOnz9PeHg4ycnJPPvss8Fukl/9+Mc/pqOjg8ceewyAz3/+8/zwhz8MbqP8xLkW\n/qpVq1xq4Q9VERERPPPMMzz22GPYbDYefPBBn1amDhXf//73OXnyJFarlfnz57NmzRqWLVsW7Gb5\nzV//+lcOHDhAWloaS5cuJSwsjJycHObNmxfspg1aXV0d69evp6uri66uLjIzM0lPTw92s4JGtcFF\nRERMzlTD4CIiIuJJYS0iImJyCmsRERGTU1iLiIiYnMJaRETE5BTWIiIiJqewFhERMTmFtYiIiMn9\nPyQ+uNKCpR6MAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0xa813090\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "# Plot the Data (Optional)\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "plt.scatter(inputs.numpy(), labels.numpy())\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JaFHyAG9nDET" + }, + "source": [ + "## Step 2: Define our TensorFlow variables\n", + "\n", + "We'll use Keras's object-oriented [`Dense`](https://www.tensorflow.org/api_docs/python/tf/contrib/keras/layers/Dense) layer to create our variables. In this case, we'll create a `Dense` layer with a single weight and bias.\n", + "\n", + "(**Note**: We're using the implementation of `Dense` found in `tf.layers.Dense` though the documentation link is for `tf.contrib.keras.layers.Dense`. When TensorFlow 1.4 is released, the documentation will also be in `tf.layers.Dense`) " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 34, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1505502830753, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "z9r-ZeyrXu3A", + "outputId": "6230a7a3-29fe-4d08-f101-da80425bad82" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 4, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Create TensorFlow Variables using Keras's Dense layer.\n", + "\n", + "wb = tf.layers.Dense(units=1, use_bias=True)\n", + "\n", + "# We can access the underlying TensorFlow variables using wb.variables.\n", + "# However, the variables won't exist until the dimensions of the input\n", + "# tensors are known. Once the dimensions of the input tensors are known,\n", + "# Keras can create and initialize the variables. Until then, Keras will\n", + "# report the variables as an empty list: [].\n", + "\n", + "wb.variables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "docKLUaonYG_" + }, + "source": [ + "## Step 3: Define our loss function\n", + "\n", + "Our loss function is the standard L2 loss (where we reduce the loss to its mean across its inputs)." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "0_w8ZJSCtuY7" + }, + "outputs": [], + "source": [ + "def loss_fn(inputs, labels, wb):\n", + " \"\"\"Calculates the mean L2 loss for our linear model.\"\"\"\n", + " predictions = wb(inputs)\n", + " return tf.reduce_mean(tf.square(predictions - labels))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 34, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 24, + "status": "ok", + "timestamp": 1505502830875, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "RkNbXoXkpjVH", + "outputId": "c36fc98d-3a57-4074-901d-c10ae017ae3f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u003ctf.Tensor: id=40, shape=(), dtype=float32, numpy=7.3549819\u003e" + ] + }, + "execution_count": 6, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Test loss function (optional).\n", + "\n", + "loss_fn(inputs, labels, wb)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 51, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 57, + "status": "ok", + "timestamp": 1505502830981, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "K_7beXoHOU7t", + "outputId": "1ad0856a-02ec-4117-a6c0-b41030981d87" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w: tf.Tensor([[ 1.56891453]], shape=(1, 1), dtype=float32)\n", + "b: tf.Tensor([ 0.], shape=(1,), dtype=float32)\n" + ] + } + ], + "source": [ + "# At this point, the variables exist, and can now be queried:\n", + "\n", + "w, b = wb.variables\n", + "print(\"w: \" + str(w.read_value()))\n", + "print(\"b: \" + str(b.read_value()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YIlebeb_qYtC" + }, + "source": [ + "## Step 4: Create our gradients function using `implicit_value_and_gradients()`\n", + "\n", + "With a loss function defined, we can calculate gradients and apply them to our variables to update them.\n", + "\n", + "To calculate the gradients, we wrap our loss function using the `implicit_value_and_gradients()` function.\n", + "\n", + "`implicit_value_and_gradients()` returns a function that accepts the same inputs as the function passed in, and returns a tuple consisting of:\n", + "\n", + "1. the value returned by the function passed in (in this case, the loss calculated by `calculate_linear_model_loss()`), and\n", + "1. a list of tuples consisting of:\n", + " 1. The value of the gradient (a `tf.Tensor`) with respect to a given variable\n", + " 1. The corresponding variable (`tf.Variable`)\n", + "\n", + "Test it out below to get a feel for what it does. Notice how the first value of the returned tuple (the loss) is the same as the value returned in the cell above that tests our loss function." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "v1spZQ4NwW1U" + }, + "outputs": [], + "source": [ + "# Produce our gradients function. See description above for details about\n", + "# the returned function's signature.\n", + "\n", + "value_and_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 153, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 46, + "status": "ok", + "timestamp": 1505502831114, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "21WMcpsmFFLd", + "outputId": "f51b3171-33f5-4f87-8bf7-0be2dc8edc8a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Outputs of value_and_gradients_fn:\n", + "Loss: tf.Tensor(7.35498, shape=(), dtype=float32)\n", + "\n", + "Gradient: tf.Tensor([[-3.00773573]], shape=(1, 1), dtype=float32)\n", + "Variable: \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e\n", + "\n", + "Gradient: tf.Tensor([-4.06519032], shape=(1,), dtype=float32)\n", + "Variable: \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e\n" + ] + } + ], + "source": [ + "# Show outputs of value_and_gradients_fn.\n", + "\n", + "print(\"Outputs of value_and_gradients_fn:\")\n", + "\n", + "value, grads_and_vars = value_and_gradients_fn(inputs, labels, wb)\n", + "\n", + "print('Loss: {}'.format(value))\n", + "for (grad, var) in grads_and_vars:\n", + " print(\"\")\n", + " print('Gradient: {}\\nVariable: {}'.format(grad, var))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JVDWpL9VYWdP" + }, + "source": [ + "## Step 5: Create an optimizer\n", + "\n", + "We'll use a `GradientDescentOptimizer` to fit our model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "DudNEebMKDWN" + }, + "outputs": [], + "source": [ + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YBeJYxY8YaiO" + }, + "source": [ + "### Step 5a: Test Our Optimizer\n", + "\n", + "Now we have everything needed to start fitting our variables to the data!\n", + "\n", + "In the next cell, we'll demo these capabilities. We'll:\n", + "\n", + "1. Print the current values of `w` and `b`\n", + "1. Calculate the loss and gradients\n", + "1. Apply the gradients\n", + "1. Print out the new values of `w` and `b`\n", + "\n", + "You can run the cell multiple times. Each time, you should see the values of `w` and `b` get closer to their true values of 3 and 2." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 102, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 103, + "status": "ok", + "timestamp": 1505502831285, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "diDZfrMJM3OC", + "outputId": "d585fff0-ecb3-4e98-9b33-bbae07a95d8c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Values of w, b, BEFORE applying gradients:\n", + "(array([[ 1.56891453]], dtype=float32), array([ 0.], dtype=float32))\n", + "()\n", + "Values of w, b, AFTER applying gradients:\n", + "(array([[ 1.86968815]], dtype=float32), array([ 0.40651903], dtype=float32))\n" + ] + } + ], + "source": [ + "# Test the optimizer.\n", + "\n", + "print(\"Values of w, b, BEFORE applying gradients:\")\n", + "w, b = wb.variables\n", + "print(w.read_value().numpy(), b.read_value().numpy())\n", + "print()\n", + "\n", + "# Calculate the gradients:\n", + "empirical_loss, gradients_and_variables = value_and_gradients_fn(\n", + " inputs, labels, wb)\n", + "optimizer.apply_gradients(gradients_and_variables)\n", + "\n", + "print(\"Values of w, b, AFTER applying gradients:\")\n", + "print(w.read_value().numpy(), b.read_value().numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "61TgeLVlKEQp" + }, + "source": [ + "## Step 6: Create a training loop\n", + "\n", + "Of course, now we can simply turn all of this code into a self-standing training loop. We'll also capture our loss and approximations of `w` and `b` and plot them over time." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 397, + "output_extras": [ + { + "item_id": 1 + }, + { + "item_id": 2 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 225, + "status": "ok", + "timestamp": 1505502831550, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "VukGe-huNaJ4", + "outputId": "f0a8d665-1910-477c-d8ab-c94ccdc4afcd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2.111051321029663, 2.3047544956207275, 2.4602210521698, 2.5850086212158203, 2.6851789951324463, 2.7655951976776123, 2.830157995223999, 2.8819968700408936, 2.9236228466033936, 2.9570505619049072]\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAFXCAYAAADnFpTQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFUbBfAzu+m9koSShBQCSC+igIAgRRGkChJEiggo\nHURAEBQBQeADRcWCha50ULFLk6IivYRQQwskhPS6O/P9sckmm4Rkk2x2difn9zz7bLuZvC8JHO7M\n7FxBkiQJREREVOlUchdARERUVTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMjArdlJQU\njB8/Hk8//TS6d++OkydPVnZdREREiiMY8znd6dOno2XLlujbty80Gg0yMzPh4uJijvqIiIgUo9TQ\nTU1NRa9evfDbb7+ZqyYiIiJFKnX38s2bN+Hp6YkZM2agd+/emD17NjIzM81RGxERkaKUGroajQbn\nzp3DoEGDsH37djg4OOCzzz4zR21ERESKUmro+vv7w9/fHw0bNgQAdO3aFefOnSvxa3g5ZyIioqJs\nShvg4+ODgIAAXL16FbVr18aRI0cQGhpa4tcIgoC4uBSTFSkHX19Xq+8BUEYfSugBYB+WRAk9AMro\nQwk9ALo+jFFq6ALArFmzMHXqVGg0GtSqVQsLFy6sUHFERERVkVGhW7duXWzdurWyayEiIlI0XpGK\niIjITBi6REREZsLQJSIiMhOGLhERkZkwdImIiMyEoUtERCbRuXM7uUuweAxdIiIyCUEQ5C7B4hn1\nOV0iIqKy+OijFTh69BAEQYUhQ4ajU6fOuH8/HnPmzER6ehq0Wi2mTJmOJ59sgwUL3kZU1HkAArp3\n74nnn39B7vIrDUOXiEhh5s6dhd27d5h0mz169MLcue8aNXbv3t9x+XI01qz5Fg8eJODll4egadNm\n+PXXn9Cq1eN48cVhkCQJmZmZOH/+POLi7uGbbzYBANLSUk1at6Xh7mUiIjKp06dP4qmnugIAPD29\n0LRpc5w/fw716j2CH37Yha+++hyXLkXD0dERtWrVwp07t7F8+RIcPXoYTk7OMldfuTjTJSJSmLlz\n3zV6VloZCq80l/e8ceOm+Oijz3H48EEsWDAXAwcOxuDBA/D11xtx9Ohh7Ny5DX/88StmzHhLjrLN\ngjNdIiIyifxwbYbff/8VoijiwYMHOHXqBOrXfwSxsbHw8PDEs8/2wrPP9sLFixeQmJgIUdSiffsn\n8fLLoxEdHSVzF5WLM10iIjKJvLOX27d/EmfPnsbQoS9AEFR49dXx8PT0wp4932PjxrWwsbGBk5Mz\nZs16G7GxsXj99TcgSSIEQcDo0eNk7qJyCVIlrThv7esjKmmNR2vvQwk9AOzDkiihB0AZfSihB8D4\n9XS5e5mIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIismjHjx/DmTOn9M93\n7NiKn3/+0STbXrv2K5Nsx1gMXSIismjHjx/D6dP5odurV1907fqMSba9Zo15Q5dXpCIiogrbsGEN\n7O3t0bfvAHzwwVJcvnwJK1Z8gmPH/sGPP+7C7NnzDMZHRV3Ahx8ug0aTDWdnN7z55hx4eXlj8+ZN\n2LlzG2xsbBAcXBujR4/Fzp1boVbb4Ndf92DixNfx779/w8nJCQMHDsa4caNQp04ETp48gczMTMya\nNRdr136FK1cuo2PHzhg5cgwAYMaMqYiLu4fs7Cz07/8CevTohVWrViI7OwvDh0eidu0QzJ49D7/8\nsgebN2+CVqtB/foNMGXKdJOuE8zQJSJSGOe5s2Bv4qX9snr0QloJiyg0btwM3367Hn37DkBU1AXk\n5ORAq9Xi1KkTaNy4mcFYjUaD5csX4733liEsrBY2bdqGTz/9CDNmvIX167/Bli27YWNjg7S0VDg7\nu+C55/rqQxYA/v33b4Pt2dra4Ysv1mDz5k2YPn0KvvpqPVxcXDFgQC8MGBAJNzc3zJw5B66ursjK\nysLIkUPQvn1HjB49Ftu2bcaXX64HAFy/fg2///4LVq36Emq1GkuXLsIvv+wx2awaYOgSEZEJRETU\nRVTUeaSnp8PW1hYREXVx/vw5nDx5HJMmTTMYGxNzHVeuXMakSa9BrVYhO1sDHx9fAEBYWDjmzn0T\n7dp1wBNPdDDqe7dt2w4AEBoahpCQUHh6egEAqlevgXv37sLNzQ3ffbcBBw7sAwDcu3cPN2/GoH79\nBgYrIv3779+4eDEKI0cOgSRJyM7OhpeXV0X/aAwwdImIFCZt7rslzkorg42NDfz9A/Djj7vQsGFj\nhIWF4/jxf3H79i0EBQUXGi0hJCQUn3zyZZFrL7///gqcOPEfDh7cjzVrvsSaNd+W+r1tbe0A6BZc\nsLW11b8uCAK0Wi2OHz+G//77F5999jXs7OwwbtwoZGdnF7MlCd26dceoUa+V40/AODyRioiITKJx\n46bYuHEdmjRphkaNmmDHjq0ID69TZFxgYDAePEjEmTOnAeh2N1+9egUAcPduLJo2bY4xY8YhLS0N\nGRnpcHJyQlpaWrnrSktLhaurK+zs7HD9+jWcPXtG/56trS20Wi0AoHnzR7F37+948OABACA5ORmx\nsbHl/r7F4UyXiIhMonHjpli79is0aNAQ9vYOsLe3L3I8F9DNit99dxGWL38fy5cvQnZ2Dp5//gXU\nqhWId96ZnRuwEvr3HwhnZxe0adMOs2a9gb/+2o+JE183OLGppJOc8t5r1ao1duzYisGDn0dgYBAa\nNGioH9OzZ2+89NJARETUxezZ8/Dyy2MwefJrEEUJtra2mDx5Gvz9/U32Z8Sl/R5CSctNWXsfSugB\nYB+WRAk9AMroQwk9AFzaj4iIyOIwdImIiMyEoUtERGQmDF0iIiIzYegSERGZCUOXiIjITBi6RERk\ndt99txFZWVlyl2F2DF0iIjK7zZs3Iisrs9j3RFE0czXmw9AlIqIK27BhDbZu1V0n+YMPlmLCBN2S\neseO/YN582YbjN2yZRPi4+MwbtxovPTSSwCAzp3bYeXK5Rg2bBDOnDmF/v17Ijk5CQBw4cJ5jBs3\nCgCQmZmJhQvfwciRL2H48ME4eHC/uVo0CV4GkohIgbyaNyj29YRjZ4p9vazjCyvL0n79+g3Et99u\nxIcfforQ0BqIi0tBZmYGGjRoiLFjJ+aOMry8Y94lHb/5ZjWaN38UM2a8hdTUVIwcOQQtWz4Ke3sH\no+qUG0OXiIgqrCxL++lIuTcdtVqN9u07Fnq/qH/+OYpDhw5g48Y1AHSLJdy9G4vAwGCT9VKZGLpE\nRApk7Ay1vOMLK9vSfkXZ2dkbLF6gVqshirrgzc7OP+FKkiS8++5i1KoVWKF65cJjukREZBLGLu0H\nAE5OzgbL9RVeeycgoDqios4DAPbt+0P/+qOPPoYtWzbpn0dHR5myhUpn1Ey3Y8eOcHFxgUqlgo2N\nDbZs2VLZdRERkZUxdmk/AOjZsxemTh2PgAB/LFmyssgSfUOHjsR7770DFxcXNG3avMDrL+ODD5bi\npZcGAgD8/QOwaNH/Kq8pEzNqab9OnTph27ZtcHd3N2qjFy9ehKdnQIWLk5OSlpuy9j6U0APAPiyJ\nEnoAlNGHEnoATLy0nyRJZfrc1IABA5CTk2P0eCIioqrAqNAVBAEjRoxA37598d1335U6/sSJE/jw\nQ+uZ7hMREZmDUcd0N23aBF9fXyQkJGDYsGEICQlBixYtHjq+Ro0aWLp0Ebp164769R8xWbFERETW\nzKhjugWtXLkSzs7OGDZs2EPH/PDDD3j22WfRvHlzHDlyBDY2/GQSERFRqWmYkZEBURTh7OyM9PR0\nHDx4EGPHji3xa7p3747nn38B3323EXPnvosJE6aYrGBzUdLBfWvvQwk9AOzDkiihB0AZfSihB8D4\nE6lKDd34+HiMHTsWgiBAq9WiR48eaNu2bakbfvfd97Bv3594//2F6NatOyIi6hpVEBERkVKVeiJV\nrVq1sHPnTuzYsQO7d+/GK6+8YtSGPTw88f77y5GdnY0JE8ZAo9FUuFgiIrJMsbF3MGTIAJNuMzr6\nIg4f/kv//ODB/Vi//huTbFuupQUr9YpU3bo9g759n8d//x3DqlUfVea3IiIimRW+wEVFXbp0EUeO\n5Idu27btEBn5kkm2XdLSgpWp0s9wmj9/Efbv34tFi95F165PP/SSYEREZN00Gg3eeWc2Ll68gNq1\nQzFr1tuwt7c3GHPr1k0sW7YYSUmJcHBwwHvvLYCLiw/++OM3fP3151Cr1XB2dsHy5R/jiy9WITs7\nG6dPn8TgwcOQlZWJCxfOYdKkaViw4G3Y2dkjOjoKiYkPMGPGW9iz53ucPXsa9es3wMyZcwAAS5a8\nh6ioc8jKykKHDp0wfPgrBksLenh4YMWKT/D330fw5ZefIScnBzVq1MTMmXPg4GD6lYsqPXS9vLyx\nePH/MGxYJCZMeBW7d/8MtVpd2d+WiKjKmjvXHrt3m/af9x49NJg7t+TdsTEx1zFjxhw0aNAQCxe+\ng+3bN2PgwMEGYxYvXoBp02aiRo2aOHfuDObOnYslS1bim2++wLJlH8HHxwdpaamwsbHByy+PRlTU\neUyc+DoAYM+e7w1m06mpKfj0069w8OA+vPHGJKxa9RVq1w7BiBEv4tKlaISFhWPUqNfg6uoKURQx\nYcIYXLlyyWBpQTc3NyQlJWLNmi+xYsXHsLd3wPr132DTpnUYOvRlk/4ZAmZaZah79x7o1asPduzY\nhs8//wSjR5d89jMREVkfPz9/NGjQEADQtesz2LLlW4PQzcjIwJkzJzF79hsFFjjQ3Tds2Bjz589B\nx46d0b79k0Z9vzZtngAAhISEwcvLG7VrhwAAatcOQWzsbYSFheP333/Grl07oNVqkZBwH1evXkVI\nSBgKLi149uwZXLt2BWPGjIAkSdBoNGjQoFHF/0CKYbYP0C5YsAQHD+7HggXvoEuXbrlNExGRqc2d\nm1XqrLQyFD6mW/gQrySJcHV1w5dfrte/lveRoalTZ+D8+bM4dOggRox4EatXryv1+9nZ2QEAVCqV\n/nHec61Wizt3bmPTpvVYvXotnJ1dsGDB2wbLBObXJaFly8cwZ867ZWm3XMy2tJ+Pjw/ee28pMjMz\nMWHCa2W6ljMREVm+2Ng7OHtWty7vr7/+jEaNmhi87+TkjICA6vjzz9/0r124cAGA7lhvvXqPYMSI\nUfDw8MS9e3fh5ORksPxfSYq7zlNaWhocHR3h5OSMhIT7OHLkkEEtedt+5JGGOH36JG7dugkAyMrK\nxI0bMWXo3HhmvVRUz5690aPHduzevQOrV3+KkSPHmPPbExFRJQoKCsa2bd9h4cK3ERwcgl69+hUZ\nM2fOu3j//YX45psvodVq0LNnD/Tv/yI+/ngFbt68AQBo3rwlwsLCUa2aH9at+xrDh0di8OCHXwUR\nKP7M6bCwcISHRyAysh+qVfNDo0aN9e/lLS3o4+OLFSs+wcyZczB37kxkZ+dAEASMHDkGtWoFVvBP\npJg6y3oZSGM97AojcXFxeOKJlsjMzMSffx7S74O3NEq6Soq196GEHgD2YUmU0AOgjD6U0ANg4qX9\nTMnX1xcLFy5Beno6Jk0ay93MRERUZZg9dAGgV6++ePrpZ3Ho0EF8/fVqOUogIiIyO1lCVxAELF78\nP3h4eOCdd97C9evX5CiDiIjIrGQJXQDw8/PD/PmLkZ6ehsmTxxV75hkREZGSyBa6ANCv3wB06dIN\nBw7sw5o1X8lZChERUaWTNXQFQcCSJSvg7u6BuXNnVdrnooiIiCyBrKELAP7+AZg3byHS0lK5m5mI\nyEoZu7Tfnj3f4/79eDNUZJlkD10AGDBgEDp16ox9+/7Ehg1r5S6HiIjKwZil/X78cTfi4uKKfa8q\nfITUIkJXEAQsXfoBXF3d8NZbM3H79i25SyIiojLKW9pv8OD+mD17epFF4vfu/R0XLpzHvHmzMXx4\nJLKystCxY0d88smHGDHiRfz5528YN24UoqJ0l4ZMSkpE//49AegC+eOPV2DkyJcwdOgg7Nq13ez9\nmYJFhC4AVK9eA++8swApKcmYMmU8dzMTEVVA8+bOxd5MNb44MTHX0afP81i3bjOcnJywfftmg/c7\ndOiEevXqY86cd/Hll+v1a+26u3tg9eq16NSpSzFb1c2ev/9+J1xcXPH559/g88+/wa5d2xEbe6dM\n9VkCiwldABg06EV06NARv//+K779doPc5RARURkUXtrv1KmTRcZIkoTCc6pOnTqXuu2//z6Cn376\nAcOGDcIrr7yE5OQkqzz51qwLHpRGEAQsW/Yh2rV7DLNnz0CHDh3h7x8gd1lERFbn2DHjVucp7/ji\nlLa038M4OjrqH6vVakiS7thudnZ2gVESJk16HS1bPlbRMmVlUTNdAKhZsxbmzJmHpKRETJ06gbuZ\niYisRGlL+wGAs7Mz0tJSH7qNgIAauHDhHAAYLAH46KOPY9u2LdBoNACAGzdikJWVacryzcLiQhcA\nhgwZhieeaI9ffvkJW7Z8K3c5RERkhLyl/QYP7o+UlORil/Z7+ulnsWTJQv2JVIVnxy+8EInt27di\n+PDBSE5O1r/eo0cvBAfXxogRgzFkyAAsWbIQWq220nsyNbMv7WesmJjraNfuMdjZ2eLAgX/g5+dn\nosqMo6Tlpqy9DyX0ALAPS6KEHgBl9KGEHgALXtrPWIGBQZg9+20kJiZi2rRJ3M1MRERWz2JDFwCG\nDXsZrVu3xZ4932PHjq1yl0NERFQhFh26KpUK//vfSjg5OWHGjKm4d++e3CURERGVm0WHLgDUrh2C\nN9+cg4SEBMyYMVXucoiIiMrN4kMXAEaMGIVWrR7H7t07rPbSX0RERFYRuiqVCitWfAQHBwdMnz4F\n8fFVd4UKIiKyXlYRugAQEhKGGTPeQnx8PGbO5G5mIiKyPlYTugDwyitj0KLFo9ixYxt++GG33OUQ\nERGViVWFrlqtxooVH8Pe3h7Tpk1CQsJ9uUsiIiIymlWFLgCEh9fBG2/MQlzcPbz55htyl0NERGQ0\nqwtdABgzZiyaNWuOrVu/w08//Sh3OUREREaxytDV7Wb+BHZ2dnj99YlITHwgd0lERESlssrQBYCI\niLp4/fUZuHs3FrNnz5C7HCIiolJZbegCwGuvTUDjxk3x7bcb8OuvP8ldDhERUYmsOnRtbGzwwQef\nwNbWFlOnTkRSUqLcJRERET2UVYcuANSrVx+TJ0/DnTu3MWfOm3KXQ0RE9FBWH7oAMH78ZDRo0Agb\nNqzFH3/8Jnc5RERExVJE6Nra2uKDDz6BjY0NJk8eh5SUZLlLIiIiKkIRoQsADRo0xMSJU3H79i3M\nnTtb7nKIiIiKUEzoAsDEiVNRv34DrF37Ffbt+1PucoiIiAwYHbqiKKJ3794YPXp0ZdZTIXZ2dvjg\ng4+hVqsxefI4pKamyF0SERGRntGhu2bNGoSGhlZmLSbRqFETjB8/CTduxGDevDlyl0NERKRnVOjG\nxsZi37596N+/f2XXYxKTJ7+BunXr4auvvsDBg/vlLoeIiAiAkaG7YMECTJs2DYIgVHY9JmFvb48V\nKz6GSqXCxIljkZaWJndJREREsCltwN69e+Hj44N69erh6NGjRm/Y19e1QoVVVJcuHTBt2jS89957\nWLZsAT744IMyb0PuHkxFCX0ooQeAfVgSJfQAKKMPJfRgLEGSJKmkAcuWLcOuXbugVquRlZWFtLQ0\ndO7cGYsXLy5xw3Fx8p/ElJmZiaeeegIXL0Zh5849ePzxNkZ/ra+vq0X0UFFK6EMJPQDsw5IooQdA\nGX0ooQfA+P84lLp7efLkydi7dy9+//13LFu2DK1atSo1cC2Fg4MDli//CCqVChMmvIr09HS5SyIi\noipMUZ/TLU6LFo9i9OixuHbtKhYunCd3OUREVIWVKXQfffRRrFq1qrJqqTRvvPEmQkPD8NlnH+Po\n0SNyl0NERFWU4me6AODo6Ijlyz8GAEyc+CoyMjJkroiIiKqiKhG6ANCq1WN45ZUxuHz5EhYtmi93\nOUREVAVVmdAFgBkz3kJwcG2sWrUS//77t9zlEBFRFVOlQtfJyQkrVnwMURQxYcKryMzMlLskIiKq\nQqpU6ALA44+3wcsvj0J09EUsWfKe3OUQEVEVUuVCFwDefHMuAgODsXLlchw/fkzucoiIqIqokqHr\n7OyM5ctX6nczZ2VlyV0SERFVAVUydAGgbdt2GDp0BC5cOI///c86rrBFRETWrcqGLgC89dY7qFUr\nECtWLMOpUyfkLoeIiBSuSoeui4srli37EFqtFuPHv4rs7Gy5SyIiIgWr0qELAO3bP4kXXxyGc+fO\nYPnyJXKXQ0REClblQxcA5s6dhxo1amL58iU4c+a03OUQEZFCMXQBuLq6YenSD6DRaDB+/Bjk5OTI\nXRIRESkQQzdXx45PYdCgF3HmzCl8+OH/5C6HiIgUiKFbwNtvz4e/fwCWLl2E06e5m5mIiEyLoVuA\nu7sHli5dgZycHAwdOhSpqalyl0RERArC0C2kc+duiIwcgv/++w8DBvRGcnKS3CUREZFCMHSL8f77\nyzFo0CD8889R9O3bEwkJ9+UuiYiIFIChWwwbGxusWbMGgwa9iJMnj6N372cRFxcnd1lERGTlGLoP\noVarsWzZhxg+fCTOnz+LXr2exp07t+Uui4iIrBhDtwQqlQoLFy7Bq6+OR3T0RfTs2Q03bsTIXRYR\nEVkphm4pBEHAnDnzMGXKG7h+/Rp69uyGK1cuy10WERFZIYauEQRBwBtvvIlZs+bi1q2beO65pxEV\ndUHusoiIyMowdMtg/PjJmD9/Ee7ejUWvXk/j9OlTcpdERERWhKFbRiNHjsGSJSuQkJCAPn2exfHj\nx+QuiYiIrARDtxyGDBmGDz9chZSUZPTt2xNHjhyWuyQiIrICDN1yev75F/DZZ18hMzMDAwf2xoED\n++QuiYiILBxDtwJ69uyNr75aD41Gg0GD+uG3336WuyQiIrJgDN0K6tr1aaxb9x1UKhVeemkQfvhh\nt9wlERGRhWLomkCHDh2xceNW2NnZ4+WXh2Dbts1yl0RERBaIoWsirVu3xebNO+Ds7IIxY17Ghg1r\n5S6JiIgsDEPXhFq0eBTbtu2Gp6cnJk58DatXfyZ3SUREZEEYuibWqFETbN/+I3x9q2HGjKn4+OMP\n5S6JiIgsBEO3EtSrVx87d+5BQEB1zJ37JpYuXQRJkuQui4iIZMbQrSRhYeHYuXMPAgODsGjRfCxY\n8A6Dl4ioimPoVqLg4NrYuXMPQkJCsWLFUsyePZ3BS0RUhTF0K1mNGjWxc+dPqFu3Hj777BNMnToR\noijKXRYREcmAoWsGfn5+2L79RzRo0Ahr136FceNGQ6PRyF0WERGZGUPXTLy9vbFt2240b94Cmzdv\nwujRI5CTkyN3WUREZEYMXTPy8PDE5s078fjjbbBr13YMHz4YmZmZcpdFRERmwtA1MxcXV2zcuBXt\n2z+Jn3/egyFDBiI9PV3usoiIyAwYujJwcnLC2rXfokuXbti79w8MGtQPqakpcpdFRESVrNTQzc7O\nRv/+/dGrVy/06NEDK1euNEddiufg4IAvv1yHHj164dChg+jfvxeSkhLlLouIiCqRTWkD7OzssGbN\nGjg6OkKr1eKFF15Au3bt0KhRI3PUp2h2dnb49NMvYW9vjy1bvkWfPj3w3Xc74O3tLXdpRERUCYza\nvezo6AhAN+vlR11My8bGBitXfooXXxyK06dPok+f7rh7967cZRERUSUodaYLAKIook+fPoiJiUFk\nZGTps9zgYHiJRa+8lHDsTLHDvZo3KPZ1WcerhCI9VGY9XwFwGDkan3++Cr16PY2tW3ejevUaFd9+\ngT6s6s+/oNweLKaeco5HzHWLqofjOd4SxisiL4CH/v0uzKjQValU2LFjB1JTU/Hqq6/i0qVLCAsL\nK/Fr1CqhyGu+vq4P+QZFx1rC+MI9VHY9n376Mby83LFo0SL07v0M/vjjDwQHB1d4+3l9yP3nWZHx\napVgUfWUZ/xDv8ZK6i843uBrLaCe8ozXP7eQeso7vrh/a+Wsp8zjoYy8MJYglfFiwCtXroSzszOG\nDRtW4ri4OOs+G9fX11WWHiRJwtKli7B48QJUr14D27btRkhIyf/BKYlcfZiSEnoA2IclUUIPgDL6\nsPgeRBHIzISQmQEh9x4ZmRCyMiFkZgKZGRAyMuE+dJBRmyt1ppuQkABbW1u4uroiMzMThw8fxiuv\nvFLhPqh4giBg6tTpcHBwxDvvzEbPnk9jy5ZdqFu3ntylERHJy8gAzHtfN7bg89z3szLzt5M7Hpm5\n28nIHZf3fna2cbWZKnTj4uIwffp0iKIIURTxzDPPoH379sYVQeU2duwEODo6YMaM19G79zP47rsd\naNiwsdxlEREVJUlAdjaE9DQI6em5tzT9PdLTIaQ95D1JA9fEFMMAzMrSPy9XAJa1fEEAHB0hOThA\ncnCE5OICyccXkoM9JAdHIO91BwdIjo6Avb3hcwcHuBj5vUoN3YiICGzfvr2CLVF5jBgxCg4Ojpg8\neRz69OmBTZu2onnzlnKXRUTWSJKAjIwioWd4nw4UfC2taEgWHZf7ulZb7tIcCpZZXAB6+0BydCga\ngA4ORQPRwQGSfe57jo4FxjoCDvaGz/O2aWsLCGU7NluYyUKX5BUZOQQODg4YO3YU+vV7Dhs2bMbj\nj7eRuywiqkyiqAuylBQIqakQUpINH6emQJWaCkg5cI5/UCQQ9Y/T8h8jIx2CCdbzllQqSE7OkJyc\nACcniN4+kJyc9K9JTk6QnAs8dnIGDN43fM+rpi/i00VdANo7AHZ2FQ5AS8bQtQJ9+z4POzt7jB49\nHAMH9sGaNZvQvv2TcpdFRAVJku64YEpKbiimFB+aqbrHKv17KbrX8h6npEBISzU6IJ2KK8XWVh9u\nors7pIDqucH38PArGJYlhSTs7U0bir6ukCz5RCoTY+haiR49noODw3oMH/4iBg9+HqtXr0GXLk/L\nXRaR9cvJyZ095oeeKi0lPwALhmZaam5gFgzRlPyvL+fFgyQ7O0iurpBcXCEGBUN0dc197gLJxS3/\nsasrJFfc0+i4AAAgAElEQVQ3iC4ukFxc4FHTDwlZAJxzw9HRUReMtram/TMik2HoWpHOnbth/frN\nGDJkIIYOjcSnn36JHj16yV0WkbxEEUJyEoTERKiSEiEkJkJISoQqMbH415ISgdRkeCcl6YKynMtr\nSmo1JBddOIoB1SE560JRdHXLD0gXV/2Y/OB0g+iS/1hycdHNHsvD1xXaKjRLVAKGrpVp164DNm3a\nhkGD+mPkyKH48MNV6N9/oNxlEVWMkcGpSnxQJECF5KQyHauUnJwAd3eInl6QagUWmUnqQzMvLAuG\npqsrRGfdPRwdFX3skSoHQ9cKPfZYa2zZshMDBvTB2LGjkJWVhcGDX5K7LKrqSgzOB/qQzAvSigan\n6O4BsXp1iPXqQ/LwgOTuAbHQveThAdHDE5KHJ0R3D0ju7oC9PXx9XfGAM0SSAUPXSjVr1gLbtn2P\n559/DpMnj0NmZgZefnm03GWRUmRkQBUfB9X9eKjux0OIj4fq/n2o7scDmalwi40zTXB6eEKsXgNi\n/UfyQ1Iflh6FXvPUvwc7u0psnqjyMHStWMOGjbBjxx707dsDM2dOQ0ZGJsaNmyh3WWSJ0tL0AaoP\n0fgCz+/H54bsfaji43UXLShB3hFIyckZoocHg5PISAxdKxcRURe7du1B3749MW/eW8jISMfrr8+A\nwGNNyiVJhiEaHwchNyzzn+cFqm52KqSnl75Ze3uI3j7QhIVD8vaG6O2ju/n4QPLxzX3uDc/QWojX\n2up21TI4icqEoasAISFh2LlTN+NdsuQ9ZGZmYvbstxm81kKSdB9FiYszDMr4eMNdvPfv658bc8at\n5OCgC9HwiEIh6gvJx0cfonnPJWcX404MqmKfqyQyJYauQgQGBmHXrp/Qt28PrFy5HBkZ6Zg/f7Hc\nZVVdkqQ7eSg2Fqo7t6G6GwukJcL5+q1Cx0lzH2dllb5JR0eIPr7Q1K2nuwpQboDqZ6O5AZoXrnB2\n5tm1RBaGoasgAQHVsWPHHvTv/xxWr/4MWVlZ+Prr1XKXpTzp6VDF3oH6bm6g6oP1DtR37kAVeweq\nu7HFzkYLXj1IcnKG6OMDTf1HdCFaIDAfGqJEZNUYugpTrVo1bN/+PQYM6IN1677BvXt3MG/eYtSu\nHSJ3aZZPo4Hq3l1daBYIT/Wd27rHsXd0AZuU+NBNSCoVRN9qutmof4D+pg2oDrewIDywdc4PUafi\nLuBHRErG0FUgLy9vbN26C6+8Mgy//PIL9u/fjwkTpmDs2ImwL++Vb6yZJEF4kKAL0oKz0dhYqGIL\nzFTj7pX4kRfRwwNiQAA0TZvlBmkARL8AiAHVIfr76+59fAGbh/y18nWFhsdCiao0hq5Cubm5Y+PG\nrfjzzz2YMGEiFi2ajy1bvsWiRcvQrl0HucsznbQ0qO8WmJnmBqsqNm+GGgvV3TslHjOVHBwg+gcg\np9XjEIsJUq2fP0T/AN0ViIiIKoChq2CCIGDAgAFo0aINFi2aj9WrP0O/fj3Rp08/vP32Qvj5+cld\n4sPlnoikvhEDJMXB4eKV/BlqgWBVJSc9fBMqFUQ/f90xU78AXaDm7uoV/fz1wSq5e/CEIyIyC4Zu\nFeDm5o758xdjwIBBmDZtErZt24Jff/0FM2fOxtChL0OtVstTWFoa1DdioI65BlXMdaivX4c6RndT\nxVyHKiVZP9S10JeKnp4Qa9SEpnkLaP0Dip2hij6+gFy9EREVg6FbhTRq1AQ//PAb1q79GvPnv40Z\nM17Hpk0b8P77/0OTJs1M/w2zs6G6dVMfpLowvaZ7fP06VPFxxX6Z5OQEbWAQcgJbQxsYBKd6dZDs\n6gWtf26g+gcADg6mr5eIqJIxdKsYtVqNoUNH4JlneuDtt2dh8+ZN6Nr1SQwdOgIzZ74Fd3cP4zcm\nirqPzsRch+r6NYNZqjrmOlR3bkMQxSJfJtnaQluzFjSPNIA2MAjawCCIuffawGBIPj4Gu3udfF2R\nxROQiEgBGLpVVLVq1fDRR59h0KAXMW3aJHz11Rf4/vtdePvt+ejb93nd1awkCUJCAtS5s1OVfvdv\n7u7gmzcgZGcX2bYkCBADqiPn0ccKhGkQxKBg3b1/AHf7ElGVxNCt4to2boIDH32OXz/7GCd3bkPW\nqyNxcdZ0NPX0hGNsLFRpqcV+nejjkztTDS4UrEHQ1qhV/kW5iYgUjKGrdFlZUF+OLjBLzdv9mzt7\nTUgAAAzOvQEAEu4jOeE+Yn184dGmLVA7JDdYdTNVba1AwMVFro6IiKwWQ1cJJAlCfDxsoqOgvhgF\ndXQUbC5GQX0pGrh9C17FXPBBsreHtlYgNI2b5odpkC5Qf4m+iCnz38btO7cReOEC3hs6Ak891VWG\nxoiIlIWha01EEapbN2Fz8QLUFy/mh2t0FFQPHhQZrq1eA2jfHhkBNQ1OVBKDgiBW8wNUqmK/Taem\nzXHwmR5YunQRPv30Iwwa1B/du/fEu+++hxo1alZ2l0REisXQtUQ5OVBfvQL1xagCs9eLsLl0sci6\nqJJKBW1wbeS0ehza8Aho6kRAWycC2vA6kFxc4evritRynPnr4uKCOXPm4fnnX8C0aZPwww+78Oef\nv2PatJkYOXI0bG1tTdUtEVGVwdCVU1oabC5dzA/V3Fmr+uoVCBqNwVDJwQHa0HBo6tTJD9fwCGhD\nQiv1pKV69epj5849+PbbDXj77VmYO/dNfPvtBixe/D+0avVYpX1fIiIlYuiagXD/ftHjrdEXob55\no8hY0d0DmibN8kO1Th1owiMg1gqU7WM2KpUKL7wwGF27Po13352Ldeu+QY8eXRAZOQSzZ78NLy9v\nWeoiIrI2DF1TkSSobt8qsEv4ItQXL8AmOgqq+/eLDNf6+SP7iQ76UNXWiYAmPAJStWoWex1gLy9v\nLFv2IQYMiMS0aZOwfv0a7NnzPd56ax4GDoyE6iHHiImISIehW1YaDdTXrhaatUZBHR1d5DOtkkoF\nMTAIWc1bFtglXAfaOhGQ3NxlaqDiWrV6DL/9th9ffPEpFi2aj4kTX8OGDWuxePH/UL/+I3KXR0Rk\nsRi6D5OeDpvTJwuEq+5sYfWVyxBycgyGSnZ20IaGI7tAqGrCI6ANDVPsNYJtbW0xZsxYPPdcb8ya\nNR3ff78TnTq1xahRr2Hq1Olw4ed4iYiKYOgCEFKSYXPqJGxOnoDNqeOwOXkCuHIZnoU+3yq6uELT\nsBG0deoW2CVcB2JQcJW9rGH16jXw5Zdr8dtvP2P69Nfx8ccfYMeOrZg/fzGeeeZZ3eUkiYgIQBUM\nXSE5CTanT+kC9uR/uvsrlw3GiG7uQLt2yKgdVuCEpgjdNYMZIsV66qmuOHCgHVasWIIPP1yOYcMi\n0blzVyxY8D6CgoLlLo+IyCIoOnSF5KQiM9giAevugewn2kPTqAk0TZoip1ETiMG14VvNrVyfb63K\nHB0dMX36bPTtOwBvvDEZv/76Mw4e3I9Jk17Hq6+Oh52dndwlEhHJSjGhW6aAbdwUmsZN9AHL2atp\nhYfXwdatu7F163eYM+dNLFjwDjZv3oRFi5ahbdt2cpdHRCQbqwzdIgF74jhsrl4xGKML2A7QNG7C\ngJWBIAjo128AOnfuioUL5+Grr75Anz7Pol+/AZg7dz6qVasmd4lERGZn8aGrD9gTx/NnsKUFbOOm\nupObGLCyc3f3wHvvLcWAAYMwbdpkbNnyLX755Se8+eYcDBkyDOoqegIaEVVNFhW6QlJi0V3EJQRs\nTpOm0DRqwoC1Ak2bNsdPP/2Br79ejQUL3sEbb0zGpk3r8P77y9GoURO5yyMiMgvZQteogPXwQHa7\nJ3Nnr00YsFZOrVZjxIhX8OyzPTFnzkxs27YFXbp0wPDhIzF9+iy4WfEFQ4iIjGGW0DUI2JPHYXvy\nONTXrhqMYcBWHX5+/li16ku88MKLmD59Cr744lPs2rUD8+YtRK9effnZXiJSrMoJ3T/+gOPev2Bz\n6oRxAdu4KcTAIAZsFdO+/ZPYu/cwVq5cjuXLl2DUqOFYv34tFi1agtDQcLnLIyIyucoJ3U6dkHcR\nQIOAzTsGy4ClXPb29pgy5Q306dMfM2ZMxR9//Ib27R/HuHGTMGHCFDgo9DKaRFQ1VU7oTp+OpPD6\nDFgyWu3aIdi4cSu+/34XZs16A0uXLsLWrd/lnvncW+7yiIhMotS12GJjYzFkyBA888wz6NGjB9as\nWVP6VhcuRHaPXjwmS2UiCAJ69HgOf/31D0aNeg03bsRg4MA+6NevH44d+wdSoWthExFZm1JDV61W\nY8aMGfjxxx+xadMmrF+/HpcvXy7ty4jKzcXFFfPmLcSvv+5HixaPYuvWrXj66U7o0OFxfPbZx0hI\nKLo+MRGRNSg1dH19fVGvXj0AgLOzM0JDQ3Hv3r1KL4yoQYOG+P77X/DTTz+hZ8/euHQpGrNmTUej\nRhEYNWoY9u/fC1EU5S6TiMhoZTqme/PmTVy4cAGNGjWqrHqIDKhUKnTt2hXNmrVGfHw8Nm/ehHXr\nvsb27VuxfftWBAUFIzJyCAYOjIS/f4Dc5RIRlUiQjDxQlpaWhhdffBGvvvoqnnrqqRLHBgej2BnI\nsWNpxY5v3ty52NflHK9SqYr0YE315ynYhyXUU57xeT3kjZckCX//fRTr13+DXbu2Iz39LADAwcER\nLi4ucHBwgCAIFlN/npgYFeKKWbnK0v/8C4/39XU16EPuesozvmAPllBPecf7+roiMLD4vT3WUD8A\ntGzpavV5Aej+fhvDqJmuRqPB+PHj8dxzz5UauHlUqqIF+Pq6PmRs8duQe3zhHuSup7zj8/qwlHrK\nM16lUhmMf/bZznj22c5ISkpCSIgKqakpyMzMQGZmBtRqNVxcXJCUdB9hYWEWUX9JX2MNf/6Fxxd8\nbAn1lGd83nNLqaf844v/AmupX/c11p8XxjJqpjtt2jR4enpixowZRm+4uP/RW5PC/5u3Vkrow9ge\nTp8+hQ0b1mDLlu+QlJQIAGjbth0iI4ege/eesn/mVwk/C0AZfSihB0AZfSihB6Dk/1QUVGpGHzt2\nDLt378aRI0fQq1cv9O7dG/v3769wgUSm1rBhIyxcuASnTkXh448/R5s2T+Dgwf0YM+ZlNGpUBzNn\nvo6zZ8/IXSYRVWFGH9MtK2v/n4uS/vdl7X1UpIcrVy5hw4Z12LhxHeLidGfdN2vWHJGRL6F3775w\ncTHuf6emoISfBaCMPpTQA6CMPpTQA2DCmS6RNQsJCcOsWXNx4sR5fPPNRnTp0g0nThzHlCnj0aBB\nHUyc+Br++ecoL7xBRGbB0KUqwdbWFk8/3R3r1n2H48fPYcaM2fDx8cWGDWvRvXtntGvXCqtWrcT9\n+7zwBhFVHoYuVTkBAdUxadLr+PvvE9i8eSd69eqDq1ev4K23ZqJRozoYOXIo9u79gxfeICKTk20R\neyK5qVQqtG//JNq3fxL379/Hli2bsH79GuzcuQ07d25DYGAQXnhhMF54YTCqV68hd7lEVMmys4H0\ndCA9XShwr3uclpb/WkZG0TEbNxr3PXgi1UMo6eC+tfdhzh4kScKxY/9g/fo12L59K9LT06BSqdCx\n41OIjHwJXbp0g62tbbm2rYSfBaCMPpTQA6CMPsrSgyiimMDLv8/IKDksC79XOEA1mvIv0GNsknKm\nS1SAIAho0eJRtGjxKObNW4gdO7Zh/fpv8Ntvv+C3336Br281DBgwCIMHD0FISNELbxCRjiQBaWlA\naqqAlBQBKSmGj9PSdI+1WiA+3v6hQVg4LE1BpZLg5AQ4OenuvbzEAs91rzk7S3B0zB9jeF/0NehX\nkS8ZZ7oPoYT/QQLK6MMSejh37iw2bFiDzZs34cGDBwCA1q3bIjJyCJ599jk4OjqWug1L6MMUlNCH\nEnoATN+HJAFZWXnhmB+SqanIDUvd4/zwzH+vuK+RpPKHpL19ySFX8N7R0XCMs/PDw9LREbC3N/2q\ns8Z+ZIih+xD8S2k5LKmHzMxM7NnzPdatW4MDB/YCANzc3NGv3/OIjHwJDRs+fDEQS+qjIpTQhxJ6\nAPL70GhQKAx1j4ubZRYOybzHea/n5JQvjezsJLi6SnBxAVxcdI9dXQFXVwnOzvmPdWN0z11cJNSs\n6YTs7LQiM0y12sR/WJWMoVtBSvtLac0stYdr165i48a12LhxPWJj7wAAGjduisjIIejTpx/c3NwN\nxltqH2WlhD4srQdJ0h2rTEwUkJgoICkp7x548CD/eeH309JUSE6Wyr3bVRAMw9DZuWAwokBA5j/P\nC1NdkOaHp719+Xq3tJ9FeTF0K0hJvwjW3oel96DRaPD7779i/fpv8OuvP0Or1cLR0RE9e/ZGZORL\naNXqMQiCYPF9GEsJfVRWD5mZKBSQMAjJwqFZ8P3sbOOD09ZWgru7BC8vFRwdtUVmjwXDMO/1ggGa\n956Tk+l3s5aVEn6fAIZuhSnpF8Ha+7CmHmJj7+Dbbzdg/fo1uHbtKgAgLCwckZEvYeTIobCzc5O5\nwoqzpp/Hw5TUQ04ODELz4YFZ9P3MTOMTTK2W4OEhwd0d8PCQ9Dd3d6nQ86Lv54Wl0n8W1oShW0FK\n+kWw9j6ssQdRFHHo0EGsW/cNfvhhF7KysgAAoaFhaN26rf4WEFBd5krLzlp+Hnlnz8bHC7h/X0B8\nvID4eBXu3xeQnm6PO3dy9KFZcBduWXbVCoIuFN3dJXh65gem4fOi73t46HbXVnSWaS0/i5IooQeA\noVthSvpFsPY+rL2HBw8SsG3bZhw48Cf27z+A1NT8XkJCQvUB3KbNE1YRwnL+PDIzDUM0Li7vsapQ\nuOoeZ2QYl2pubg+bZepC82GzUFfXsq+nakrW/ncDUEYPAEO3wpT0i2DtfSihB0DXx507D3DmzCn8\n9ddBHDp0AEeOHEZKSrJ+TO3aIWjT5gk8/ngbtGnzhEVeCcuUP4+cHCAhQReexYWmLlhV+sepqaWH\nqL29BB8fw5u3twQfH1H/PDTUCZKUCg8PCW5ugI2VXrFACX83lNADwNCtMCX9Ilh7H0roASi+D61W\naxDChw8fMgjh4ODaBiFco0ZNc5ddREk/D61Wd7Zt4QDNn5EWfE+FxMTSQ9TGJi808wPU17domOa9\n7uxc+m5bJf9OWRsl9AAwdCtMSb8I1t6HEnoAjOtDq9Xi7NnTBiGcnJykfz8oKNgghGvWrFXZZUOj\nAe7dE3DnjoDYWBUyMx1x7VpWkRlpfLyAhAQBolhy4qlUEry8Cs9Ciz7OC1N398q5kEFV+Z2ydEro\nAWDoVpiSfhGsvQ8l9ACUrw+tVotz587gr78O4NChgzh8+BCSkhL17wcGBqNNm/wTs2rVCizT9tPS\ngNhYAbdvq/SheueOgNu38x/fu1d6kHp46EKy5Bmp7ubpKcl+4YOq/DtlaZTQA8DQrTAl/SJYex9K\n6AEwTR+6ED6LQ4cO4K+/DuLw4b8KhXAQWrdui8cea4v69dtDrQ7MDVEVYmMF3LmjC1LdTYXk5IeH\nqZ2dBH9/CQEBIgICJAQESPD3FxEW5gBb23T4+OhC1ctLQjnXgJANf6cshxJ6AIwPXSs9fYCoalKr\n1ahTpxHc3BqjcePx6NVLwokT93DqVDyuXMnErVu22LTJD5s21QBg99DtuLtLqFFDRPPmulD195dQ\nvXr+44AA3ey0uN26vr4OiIvTVl6TRArG0CWyEJIEJCejwK5e3Wy04K7e2FjdCUiGaufedMdLfXyy\n4eAQj5yca0hMPI2srCsAbgG4CT8/EW3ahKB9+1Zo3botAgODIMh9SSKiKoShS2QGWi1w6xZw5oyq\nwK7egrt7da+VdGEGJyfdDLRuXU3uzFTM3eWrm6FWr67b3as7XuoKoCFE8RGcP38Ohw8fxF9/peDw\n4YPYtu0Atm37BgBQo0ZN/WeEW7dui6CgYIYwUSXiMd2HUNJxBmvvw1p6SEkBrl9X4do1Fa5fF3D9\nukr//ObNkldv8fExPG4aEKAL1bxdvQEBItzcKn4WryiKuHDhfG4IH8Thwwdx//59/fs1atTUnxnd\nunVbBAfXLhLC1vLzKIkSegCU0YcSegB4IlWFKekXwdr7sJQetFrdmb7Fher16wLu3y/+0kQ+PiKC\ngiSEhqrh5ZVtcGJSQIAIP7/yr9BSUaIoIirqAg4dOoBDh/7CoUMHDEK4evUaBiFcu3YIqlVzs4if\nR0VYyu9URSmhDyX0ADB0K0xJvwjW3oc5e0hNhT5M84JVF6oq3LhR/EowtrYSAgMlBAWJ+ltwcP5z\nFxfz91FekiQhKuoC/vrrAA4f1oVwfHy8/n1//wA0bdoEgYEhqFMnAuHhEQgPrwNvb28Zqy47a/hZ\nGEMJfSihB4BnLxMVSxR1s9W8UL12LT9Ur18v7iQlHW9vEQ0aFAxV3ew1KEg3a5X7c6emIggC6tat\nh7p162HEiFcgSRIuXozSh/CRI4ewZ8+eIl/n7e2tD+D8WwRq1qwFlZwXJyayMAxdUpz0dBjMVAvu\nAo6JUSErq+hs1cZGQq1aEho00BQJ1aAg3fHUqkgQBERE1EVERF0MHz4SAGBjo8GRI/8hOvoiLl6M\nwqVLuvu//z6CI0cOGXy9o6MjQkPDUadOnQKhHIGQkFDYy7VPnUhGDF2yOpIE3L1reGy14Gz13r3i\nZ1aenhLq1Ss6Uw0K0p35a60XvTc3T09PtGjxKFq0eNTg9czMTFy9egXR0VEFwvgiLl+OxpkzpwzG\nqlQqBAUFo06dCISF1cndVa2bIbu7e5izHSKz4j8zZJEkSbcb+MIFFWJjgTNn7PWhGhOjKnbJNrVa\nQs2aEtq10+hDVXevu7m7y9BIFeLg4IB69eqjXr36Bq+LooibN2/khvFF/cw4OjoKP/+8Bz//bLi7\nulo1v9wwDjc4bhwQUJ0fZyKrx9AlWUmS7mL6Fy6oEBWlu124oMbFiyokJRX8B1Z3dSU3Nwnh4WKB\nMJX0M9caNThbtUQqlQqBgUEIDAxCp05dDN67f/8+oqOj9Luqo6OjcOlSNA4e3I+DB/cbjHV2dkF4\neDjCwyMMZsjBwbVha23XoaQqi/9EkdnExeWHa37Iqoss76ZWSwgJEfHEEyIiIkS0bGkPb+80BAWJ\n8OCeR0Xx9vaGt3drPPZYa4PX09PTcflydIEw1s2Qz507ixMnjhuMtbGxQe3aIQXCOFx/7+Ji3Bml\nRObC0CWTu39fKBSsulvhz7GqVBJq15bQurUGdevqAjYiQkRoqGjwuVVfX3vExYlm7oLk5OTkhIYN\nG6Nhw8YGr2s0GsTEXEN0dLTBSVzR0RcRHX0RP/6422B89eo1DM6mzpsh+/i4mLMdIj2GLpXbgwdA\nVJS60K5hVZGP3QiChKAgCS1b5hiEa1iYCAcHmYonq2RjY4OQkDCEhISha9en9a9LkoR79+4VOYkr\nOjoK+/b9iX37/jTYjouLC/z9A+DvHwA/P//cez+D1/z8/OHk5GTuFknhGLpUqqQk4MIFtUGwRkWp\nij1LODBQRJcuGkREaBERIaJuXV248t8uqkyCIMDPzw9+fn5o27adwXupqSkFPt6kmyHfvHkdt2/f\nxqVL0SVu193dA/7+/vDzC4C/v39uKPvnhnL+Y378iYzF0CW9lBTkBqraIFxjY4uGa61aIp56SpM7\na9Wibl0R4eEinJ1lKJyoBC4urmjatDmaNm2ufy3vKkhZWVm4d+8u7t6NRWxsLO7evYPY2FjExt5B\nbOyd3NfvICrqQonfw8vLq5hgDjAI6WrV/HjCFzF0q6LUVODixfwzhfPC9fbtouFao4aIjh01ubNW\n3ey1Tp38SxsSWTN7e3vUqhWIWrUCSxyXkZGBe/fuFgjm/HDOC+Zbt27i/PmzD92GIAjw9vbRB3HB\nXdsFw9nHxxc2PA1fsfiTVbD0dODff4HDh230s9eoKBVu3CgargEBIjp00Oh3CeftHnblyZ9EcHR0\nRFBQMIKCgkscl5aWhrt3Y/VBnB/M+Y+vXLlc5GIhBalUKvj6Vis0Yy46g7a2612TDkNXITQa3a7h\n48fVOH5chf/+081gRREAHPXjqlUT8cQT+WcL581eeeEIoopzdnZGSEgoQkJCSxyXmppisBu74K7t\n/F3a53Hy5PGHbsPGxgZeXl5wc3OHu7s73N09Ctx76J97eHjAza3ovVopFwy3MgxdKyRJQEyMgOPH\n1fjvP13InjqlNrhKk5OThJYttWjZ0gaBgZn62aunp4yFExEA3XHmsDBXhIWFP3SMJElITk4q9hjz\n3bt3ERt7B8nJiUhIeICYmOvIzs4uUw2urm7FhLW7QVjnv+ZpEOCOjo68Olg5MXStQEICcOJEXsDq\nQrbgx3JUKgl164po1kyLZs1ENG2qm73a2OSdMJIjY/VEVB6CIOhnrBERdYsdk3dCmCRJyMjIQHJy\nEhITE5GUlISkpAe597rniYmJ+vcL3sfEXEdKSnKZarOzs9PPmjnLLhuGroXJyABOn87bTawL2mvX\nDI/BBgaKeO65HDRtqgvZhg21PGuYqAoTBAFOTk5wcnKCv39Amb9eq9UiOTnJIKSTkhILBHhigZth\nkF+/fg05OWX7j72rq5s+gH18vGBraw9HR139jo6OcHJy1t87ORV87lRgXP69s7Pu3hrCnKErI60W\niI5W4b//VPpZ7PnzKmg0+bttPD0ldOyoyQ1YLZo0EeHrK8lYNREpjVqthqenFzw9vcr8tWWdZRcM\n7piY6zh79rTJ+rC3ty8S2oXDOu/2sJA3DHvDcLezs6vwbnWGrplIEnD7tqA/Bnv8uBonTqiRlpb/\nA7S3l9CkiW43cdOmulvt2hJ46ISILFVFZ9ne3s6IibmH9PR0ZGSkF3ufd8vIyEB6elqh+4LjdK+l\npaUjOTkZsbGxyMhIhyia5jKyarW6SFjnzcT3799r1DZKDd2ZM2di79698Pb2xu7du0sbTrmSkqDf\nRZx3NnHBKzgJgoSICBFNm4r6WWzduiLs7GQsmojIzFQqFZydneFcScfIJElCVlZWgSDXBXZ6enEB\nXtmry4cAAAsRSURBVFyQFwx9w/vExESkp6eVafd6qaHbp08fvPjii5g2bVqFGleyrCzg7FmVwdnE\nly4ZHluoXl1E9+45aNpUN5Nt3FjLz8ASEVUyQRDg4OAABweHcu0+N4ZJQ7dFixa4detWhQpSElEE\nLl/WHYfNm8meOaNCTk7+PmBXV91C6rrdxLqZrL8/j8MSESlRWS7vyWO6pbh7V3ccNu9kpxMn1EhJ\nyQ9YW1sJDRqI+mOwzZrplqZTFb3oExERVXEM3QIkCTh/XoUDB9Q4fhw4csS5yPWIw8K06NYt/2Sn\nRx4xXPuViIjoYSotdH19reOA5Y0bwG+/6W6//w7cvZv/np+fCj17Ao8+CrRqBbRoAXh4qAGoAVjP\naiHW8rMoiRJ6ANiHJVFCD4Ay+lBCD8YyKnQlqezHI+PiUsr8NeaQlAQcPGiD/fvV2L/fBpcv589k\nq1UT0a+fFu3aadCzpyMcHVMMPq6TkwPExclQdAXkXbHGmimhB4B9WBIl9AAoow8l9AAY/x+HUkN3\nypQpOHr0KBITE9GhQweMGzcOffv2rXCB5pKVBfzzj1ofsidOqCCKuiR1dpbQpYsG7dpp0K6d7tKJ\neSHr62t9AUtERJat1NBdunSpOeowGVHUfXxn3z5dyB49mr8QgI2NbhGAdu10t2bNtOCa0kREZC6K\nOJHq+nUB+/frdhkfOKBGQkL+LuN69fJCVoPHH9dy8XUiIpKNVYZuQoLuuGzebPb69fyQrV5dxMCB\nOWjXToMnntDCz4+fjyUiIstgFaGbkQEcPZp/XPb0aRUkSbfL2M1NwjPP5Ohns6GhvFYxERFZJosM\nXa0WOHVKpd9l/PffamRl6ZLUzk5Cmzb5u4wbNdKtG0tERGTpLCKuJAm4elXAvn26kD140AZJSfnT\n1YYN80O2VSstnJxkLJaIiKicZAvde/cEHDyYv8v45s3847KBgSJ69tTtMm7TRgsfHx6XJSIi62e2\n0E1NBY4cUetns+fP56/C4+kp6UO2XTsNgoMZskREpDyVFro5OcDx4/nHZf/9Vw2NRrfL2MFBQvv2\nugtStG+vQYMGXCCAiIiUr1JCt2dP4M8/XZCaqgtZQZDQpImov/JTy5ZaODhUxncmIiKyXJUSurt3\nAyEhEvr10+0ybttWAw+PyvhORERE1qNSQvfaNcDJKa0yNk1ERGS1KuVIalBQZWyViIjIuvH0JSIi\nIjNh6BIREZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIR\nEZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eI\niMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMGLpE\nRERmwtAlIiIyE4YuERGRmRgVuvv370e3bt3QtWtXfPbZZ5VdExERkSKVGrqiKGLevHlYvXo1vv/+\ne/zwww+4fPmyOWojIiJSlFJD99SpUwgKCkKNGjVga2uL7t274/fffzdHbURERIpSaujevXsXAQEB\n+ud+fn64d+9epRZFRESkRKWGriRJ5qiDiIhI8WxKG+Dv74/bt2/rn9+9exfVqlUrdcO+vq4Vq8wC\nKKEHQBl9KKEHgH1YEiX0ACijDyX0YKxSZ7oNGzZETEwMbt26hezsbPzwww/o1KmTOWojIiJSlFJn\numq1GrNnz8bw4cMhSRL69euH0NBQc9RGRESkKILEg7ZERERmwStSERERmQlDl4iIyEwYukRERGZS\n6olUZbF//34sWLAAkiShb9++eOWVV0y5ebOYOXMm9u7dC29vb+zevVvucsolNjYW06ZNQ3x8PNRq\nNfr3748hQ4bIXVaZZWdnIzIyEjk5OdBqtejatSvGjh0rd1nlIooi+vbtCz8/P6xatUrucsqlY8eO\ncHFxgUqlgo2NDbZs2SJ3SeWSkpKCN998E9HR0VCpVFiwYAEaN24sd1lGu3r1KiZNmgRBECBJEm7c\nuIEJEyZY5d/xr7/+Glu2bIEgCKhTpw4W/r+9u3mJag8DOP6dHKRQexElCyzIjCySFr1AEyamSTXV\nxGCLNiVRbdIow14oghYJLfoHWkREEBEaRG1EszGmQiuGYIgwIhhMKkRT5yXPnOcu4l64G+89x7nz\na7rPZz1n+A6HmYcznHmmo4P8/HzTWY7cunXrr/fCv/qslQxJp9NSX18vsVhMfvz4IXv37pWhoaFM\nPX3WDAwMSDQaFb/fbzrFtS9fvkg0GhURkcnJSdmxY0dOngsRkXg8LiIilmVJU1OTRCIRw0Xu3Lx5\nU9ra2uT48eOmU1yrq6uTsbEx0xmzdvbsWbl//76IiExPT8vExIThIvfS6bT4fD4ZHh42neLYyMiI\n1NXVSSqVEhGRkydPSldXl+EqZ96/fy9+v19SqZRYliWHDx+WT58+zXhMxr5e/l12NG/YsIH58+eb\nzpiV0tJSqqqqACgoKKCioiJnV3fOmzcP+HnVa1mW4Rp3RkZGePr0KU1NTaZTZkVEsG3bdMasTE5O\nMjg4SDAYBMDr9VJYWGi4yr1wOMyyZcv+tqo3l9i2TSKRwLIsksnkv1q89Cv58OED69evJz8/n7y8\nPDZu3Eh3d/eMx2Rs6OqO5l9TLBbj3bt3VFdXm05xxbZtAoEAPp8Pn8+Xk6/j6tWrtLe34/F4TKfM\nisfj4ciRIwSDQe7du2c6x5VYLMaiRYs4f/48+/fv59KlSySTSdNZrj1+/Jjdu3ebznBl8eLFNDc3\nU1tbS01NDUVFRWzZssV0liOVlZUMDAwwPj5OIpEgFArx+fPnGY/J2NAV/bnvL2dqaorW1lYuXLhA\nQUGB6RxX5syZw4MHDwiFQkQiEYaGhkwnOdLX10dJSQlVVVU5/x65e/cunZ2d3Lhxgzt37jA4OGg6\nyTHLsohGoxw8eJCuri7mzp2bs/8RPj09TW9vLzt37jSd4sr379/p6enhyZMn9Pf3E4/Hc+4+moqK\nCo4ePUpzczPHjh1j9erVeL0z3yqVsaHrdkez+m9YlkVrayv79u2jvr7edM6sFRYWsmnTJvr7+02n\nOPL69Wt6e3vZvn07bW1tvHz5kvb2dtNZrpSWlgJQXFxMQ0MDb9++NVzkXFlZGWVlZaxbtw6AxsZG\notGo4Sp3QqEQa9eupbi42HSKK+FwmPLychYuXEheXh4NDQ28efPGdJZjwWCQzs5Obt++zYIFC1i+\nfPmMj8/Y0P2ddjTn+hUJ/LwLe+XKlRw6dMh0imujo6NMTEwAkEwmef78OStWrDBc5czp06fp6+uj\np6eH69evs3nzZq5du2Y6y7FEIsHU1BQA8XicZ8+eUVlZabjKuZKSEpYsWcLHjx8BePHiRc6utX30\n6BF+v990hmtLly4lEomQSqUQkZw9F6OjowAMDw/T3d39j+ckYz8Z+l12NP95NTI2NkZtbS0tLS1/\n3XSRK169esXDhw9ZtWoVgUAAj8fDqVOnqKmpMZ3myNevXzl37hy2bWPbNrt27WLbtm2ms/6Xvn37\nxokTJ/B4PKTTafbs2cPWrVtNZ7ly8eJFzpw5g2VZlJeX09HRYTrJsWQySTgc5sqVK6ZTXKuurqax\nsZFAIIDX62XNmjUcOHDAdJZjLS0tjI+P4/V6uXz5MkVFM/9jku5eVkoppbJEN1IppZRSWaJDVyml\nlMoSHbpKKaVUlujQVUoppbJEh65SSimVJTp0lVJKqSzRoauUUkpliQ5dpZRSKkv+AO2e4yf8wTuC\nAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0xc1dc310\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "# Train our variables.\n", + "\n", + "# numpy is used for its asscalar() function.\n", + "import numpy as np\n", + "\n", + "num_training_steps = 10\n", + "\n", + "def train_model(inputs, labels, wb, optimizer, num_training_steps):\n", + " loss_at_step = []\n", + " w_at_step = []\n", + " b_at_step = []\n", + " for step_num in range(num_training_steps):\n", + " loss, gradients_and_variables = value_and_gradients_fn(inputs, labels, wb)\n", + " loss_at_step.append(np.asscalar(loss.numpy()))\n", + " \n", + " optimizer.apply_gradients(gradients_and_variables)\n", + " w, b = wb.variables\n", + " w_at_step.append(np.asscalar(w.read_value().numpy()))\n", + " b_at_step.append(np.asscalar(b.read_value().numpy()))\n", + "\n", + " print(w_at_step)\n", + " t = range(0, num_training_steps)\n", + " plt.plot(t, loss_at_step, 'k',\n", + " t, w_at_step, 'r',\n", + " t, [true_w] * num_training_steps, 'r--',\n", + " t, b_at_step, 'b',\n", + " t, [true_b] * num_training_steps, 'b--')\n", + " plt.legend(['loss', 'w estimate', 'w true', 'b estimate', 'b true'])\n", + " plt.show()\n", + "\n", + "train_model(inputs, labels, wb, optimizer, num_training_steps)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "UNurY9VJ-hpH" + }, + "source": [ + "## Other Ways to Compute Gradients\n", + "\n", + "Using our loss function as an example (`calculate_linear_model_loss()`), there are several other ways we could compute gradients:\n", + "\n", + "1. `tfe.implicit_gradients()`\n", + "1. `tfe.gradients_function()`\n", + "1. `tfe.implicit_value_and_gradients()`\n", + "1. `tfe.value_and_gradients_function()`\n", + "\n", + "Each of these functions does the following:\n", + "* Wraps a function.\n", + "* Returns a function with the same input signature as the wrapped function.\n", + "\n", + "They differ only in what information they return.\n", + "\n", + "### Gradients-only functions\n", + "\n", + "The following two functions return a function that returns only the variables' gradients:\n", + "\n", + "1. `tfe.gradients_function()`: Returns the partial derivatives of the function `f()` with respect to the parameters of `f()`.\n", + "1. `tfe.implicit_gradients()`: Returns the partial derivatives of the function `f()` with respect to the trainable parameters (`tf.Variable`) used by `f()`.\n", + "\n", + "In our example above, the `tf.layers.Dense` object encapsulates the trainable parameters.\n", + "\n", + "### Value and gradients functions\n", + "\n", + "The following two functions are identical to their counterparts above, except that they also return the value of the wrapped function.\n", + "\n", + "1. `tfe.implicit_value_and_gradients()`\n", + "1. `tfe.value_and_gradients_function()`\n", + "\n", + "### Gradient demos\n", + "\n", + "In the demos below, we show examples for the `implicit_*` functions, since our existing loss function works seamlessly with these versions. (The other versions require that your parameters are tensors and tensors only; in our example, we're using a `Dense` layer.)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 85, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 100, + "status": "ok", + "timestamp": 1505502831671, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "aEoCftnfAIH5", + "outputId": "72f1c1dc-a574-463f-f860-c4e5f48fcdaa" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[(\u003ctf.Tensor: id=673, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", + " (\u003ctf.Tensor: id=671, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)]" + ] + }, + "execution_count": 13, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# tfe.implicit_gradients() demo\n", + "gradients_fn = tfe.implicit_gradients(loss_fn)\n", + "\n", + "# Returns only gradients and variables:\n", + "gradients_fn(inputs, labels, wb)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 102, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 88, + "status": "ok", + "timestamp": 1505502831785, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "bbgCUdCzAVhH", + "outputId": "152aa9b6-9e42-4b7e-848a-9423c0b1929c" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(\u003ctf.Tensor: id=688, shape=(), dtype=float32, numpy=1.0623235\u003e,\n", + " [(\u003ctf.Tensor: id=720, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", + " (\u003ctf.Tensor: id=718, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)])" + ] + }, + "execution_count": 14, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# tfe.implicit_value_and_gradients() demo\n", + "value_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)\n", + "\n", + "# Returns only gradients:\n", + "value_gradients_fn(inputs, labels, wb)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Eager Execution Tutorial: Working with Gradients", + "provenance": [], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb new file mode 100644 index 0000000000..ebcc7027c1 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U9i2Dsh-ziXr" + }, + "source": [ + "# Eager Execution Tutorial: Importing Data\n", + "\n", + "This notebook demonstrates the use of the [`tf.contrib.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n", + "\n", + "* Creating a `Dataset`.\n", + "* Iteration over a `Dataset` with eager execution enabled.\n", + "\n", + "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n", + "\n", + "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly different. You will use a Pythonic `Iterator()` class instead of using `make_one_shot_iterator()` and `get_next()`. As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z1JcS5iBXMRO" + }, + "source": [ + "# Setup: Enable eager execution\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RlIWhyeLoYnG" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "import tensorflow.contrib.eager as tfe\n", + "\n", + "# Enable eager execution\n", + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H9UySOPLXdaw" + }, + "source": [ + "# Step 1: Create a source `Dataset`\n", + "\n", + "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "WPTUfGq6kJ5w" + }, + "outputs": [], + "source": [ + "ds_tensors = tf.contrib.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", + "\n", + "# Create a CSV file\n", + "import tempfile\n", + "_, filename = tempfile.mkstemp()\n", + "with open(filename, 'w') as f:\n", + " f.write(\"\"\"Line 1\n", + "Line 2\n", + "Line 3\n", + " \"\"\")\n", + "ds_file = tf.contrib.data.TextLineDataset(filename)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "twBfWd5xyu_d" + }, + "source": [ + "# Step 2: Apply transformations\n", + "\n", + "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.contrib.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset) for details." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ngUe237Wt48W" + }, + "outputs": [], + "source": [ + "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", + "ds_file = ds_file.batch(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IDY4WsYRhP81" + }, + "source": [ + "# Step 3: Iterate\n", + "\n", + "Use `tfe.Iterator` on the `Dataset` object to get a Python iterator over the contents of the dataset.\n", + "\n", + "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that this process of iteration is different. Here there are no calls to `Dataset.make_one_shot_iterator()` and no `get_next()` calls." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 153, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 201, + "status": "ok", + "timestamp": 1505952405928, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "lCUWzso6mbqR", + "outputId": "ec027d30-96c6-4ea4-9ee1-ef74ec1ae29a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Elements of ds_tensors:\n", + "tf.Tensor([4 9], shape=(2,), dtype=int32)\n", + "tf.Tensor([16 25], shape=(2,), dtype=int32)\n", + "tf.Tensor([36 1], shape=(2,), dtype=int32)\n", + "\n", + "Elements in ds_file:\n", + "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n", + "tf.Tensor(['Line 3' ' '], shape=(2,), dtype=string)\n" + ] + } + ], + "source": [ + "print('Elements of ds_tensors:')\n", + "for x in tfe.Iterator(ds_tensors):\n", + " print(x)\n", + "\n", + "print('\\nElements in ds_file:')\n", + "for x in tfe.Iterator(ds_file):\n", + " print(x)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Eager Execution Tutorial: Importing Data", + "provenance": [], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD new file mode 100644 index 0000000000..5759ca17fa --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/BUILD @@ -0,0 +1,43 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_library( + name = "resnet50", + srcs = ["resnet50.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + ], +) + +cuda_py_test( + name = "resnet50_test", + size = "large", + srcs = ["resnet50_test.py"], + additional_deps = [ + ":resnet50", + "//tensorflow/contrib/summary:summary_test_util", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) + +cuda_py_test( + name = "resnet50_graph_test", + size = "large", + srcs = ["resnet50_graph_test.py"], + additional_deps = [ + ":resnet50", + "//tensorflow/contrib/summary:summary_test_util", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + ], + tags = [ + "noasan", + "nomsan", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/resnet50/README.md b/tensorflow/contrib/eager/python/examples/resnet50/README.md new file mode 100644 index 0000000000..f6c1defa42 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/README.md @@ -0,0 +1,34 @@ +Image classification using the ResNet50 model described in +[Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385). + +Contents: + +- `resnet50.py`: Model definition +- `resnet50_test.py`: Sanity unittests and benchmarks for using the model with + eager execution enabled. +- `resnet50_graph_test.py`: Sanity unittests and benchmarks when using the same + model code to construct a TensorFlow graph. + +# Benchmarks + +Using a synthetic data. + +``` +# Using eager execution +bazel run -c opt --config=cuda :resnet50_test -- --benchmarks=. + +# Using graph execution +bazel run -c opt --config=cuda :resnet50_graph_test -- --benchmarks=. +``` + +(Or remove the `--config=cuda` flag for running on CPU instead of GPU). + +On October 31, 2017, the benchmarks demostrated comparable performance +for eager and graph execution of this particular model when using +a single NVIDIA Titan X (Pascal) GPU on a host with an +Intel Xeon E5-1650 CPU @ 3.50GHz and a batch size of 32. + +| Benchmark name | batch size | images/second | +| --------------------------------------- | ------------- | ------------- | +| eager_train_gpu_batch_32_channels_first | 32 | 171 | +| graph_train_gpu_batch_32_channels_first | 32 | 172 | diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py new file mode 100644 index 0000000000..b302a87e0e --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py @@ -0,0 +1,324 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet50 model definition compatible with TensorFlow's eager execution. + +Reference [Deep Residual Learning for Image +Recognition](https://arxiv.org/abs/1512.03385) + +Adapted from tf.keras.applications.ResNet50. A notable difference is that the +model here outputs logits while the Keras model outputs probability. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import tensorflow as tf +import tensorflow.contrib.eager as tfe + + +class _IdentityBlock(tfe.Network): + """_IdentityBlock is the block that has no conv layer at shortcut. + + Args: + kernel_size: the kernel size of middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + data_format: data_format for the input ('channels_first' or + 'channels_last'). + """ + + def __init__(self, kernel_size, filters, stage, block, data_format): + super(_IdentityBlock, self).__init__(name='') + filters1, filters2, filters3 = filters + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + bn_axis = 1 if data_format == 'channels_first' else 3 + + self.conv2a = self.track_layer( + tf.layers.Conv2D( + filters1, (1, 1), + name=conv_name_base + '2a', + data_format=data_format)) + self.bn2a = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')) + + self.conv2b = self.track_layer( + tf.layers.Conv2D( + filters2, + kernel_size, + padding='same', + data_format=data_format, + name=conv_name_base + '2b')) + self.bn2b = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')) + + self.conv2c = self.track_layer( + tf.layers.Conv2D( + filters3, (1, 1), + name=conv_name_base + '2c', + data_format=data_format)) + self.bn2c = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')) + + def call(self, input_tensor, training=False): + x = self.conv2a(input_tensor) + x = self.bn2a(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2b(x) + x = self.bn2b(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2c(x) + x = self.bn2c(x, training=training) + + x += input_tensor + return tf.nn.relu(x) + + +class _ConvBlock(tfe.Network): + """_ConvBlock is the block that has a conv layer at shortcut. + + Args: + kernel_size: the kernel size of middle conv layer at main path + filters: list of integers, the filterss of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + data_format: data_format for the input ('channels_first' or + 'channels_last'). + strides: strides for the convolution. Note that from stage 3, the first + conv layer at main path is with strides=(2,2), and the shortcut should + have strides=(2,2) as well. + """ + + def __init__(self, + kernel_size, + filters, + stage, + block, + data_format, + strides=(2, 2)): + super(_ConvBlock, self).__init__(name='') + filters1, filters2, filters3 = filters + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + bn_axis = 1 if data_format == 'channels_first' else 3 + + self.conv2a = self.track_layer( + tf.layers.Conv2D( + filters1, (1, 1), + strides=strides, + name=conv_name_base + '2a', + data_format=data_format)) + self.bn2a = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')) + + self.conv2b = self.track_layer( + tf.layers.Conv2D( + filters2, + kernel_size, + padding='same', + name=conv_name_base + '2b', + data_format=data_format)) + self.bn2b = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')) + + self.conv2c = self.track_layer( + tf.layers.Conv2D( + filters3, (1, 1), + name=conv_name_base + '2c', + data_format=data_format)) + self.bn2c = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')) + + self.conv_shortcut = self.track_layer( + tf.layers.Conv2D( + filters3, (1, 1), + strides=strides, + name=conv_name_base + '1', + data_format=data_format)) + self.bn_shortcut = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '1')) + + def call(self, input_tensor, training=False): + x = self.conv2a(input_tensor) + x = self.bn2a(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2b(x) + x = self.bn2b(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2c(x) + x = self.bn2c(x, training=training) + + shortcut = self.conv_shortcut(input_tensor) + shortcut = self.bn_shortcut(shortcut, training=training) + + x += shortcut + return tf.nn.relu(x) + + +class ResNet50(tfe.Network): + """Instantiates the ResNet50 architecture. + + Args: + data_format: format for the image. Either 'channels_first' or + 'channels_last'. 'channels_first' is typically faster on GPUs while + 'channels_last' is typically faster on CPUs. See + https://www.tensorflow.org/performance/performance_guide#data_formats + name: Prefix applied to names of variables created in the model. + trainable: Is the model trainable? If true, performs backward + and optimization after call() method. + include_top: whether to include the fully-connected layer at the top of the + network. + pooling: Optional pooling mode for feature extraction when `include_top` + is `False`. + - `None` means that the output of the model will be the 4D tensor + output of the last convolutional layer. + - `avg` means that global average pooling will be applied to the output of + the last convolutional layer, and thus the output of the model will be + a 2D tensor. + - `max` means that global max pooling will be applied. + classes: optional number of classes to classify images into, only to be + specified if `include_top` is True. + + Raises: + ValueError: in case of invalid argument for data_format. + """ + + def __init__(self, + data_format, + name=None, + trainable=True, + include_top=True, + pooling=None, + classes=1000): + super(ResNet50, self).__init__(name='') + + valid_channel_values = ('channels_first', 'channels_last') + if data_format not in valid_channel_values: + raise ValueError('Unknown data_format: %s. Valid values: %s' % + (data_format, valid_channel_values)) + self.include_top = include_top + + def conv_block(filters, stage, block, strides=(2, 2)): + l = _ConvBlock( + 3, + filters, + stage=stage, + block=block, + data_format=data_format, + strides=strides) + return self.track_layer(l) + + def id_block(filters, stage, block): + l = _IdentityBlock( + 3, filters, stage=stage, block=block, data_format=data_format) + return self.track_layer(l) + + self.conv1 = self.track_layer( + tf.layers.Conv2D( + 64, (7, 7), + strides=(2, 2), + data_format=data_format, + padding='same', + name='conv1')) + bn_axis = 1 if data_format == 'channels_first' else 3 + self.bn_conv1 = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name='bn_conv1')) + self.max_pool = self.track_layer( + tf.layers.MaxPooling2D((3, 3), strides=(2, 2), data_format=data_format)) + + self.l2a = conv_block([64, 64, 256], stage=2, block='a', strides=(1, 1)) + self.l2b = id_block([64, 64, 256], stage=2, block='b') + self.l2c = id_block([64, 64, 256], stage=2, block='c') + + self.l3a = conv_block([128, 128, 512], stage=3, block='a') + self.l3b = id_block([128, 128, 512], stage=3, block='b') + self.l3c = id_block([128, 128, 512], stage=3, block='c') + self.l3d = id_block([128, 128, 512], stage=3, block='d') + + self.l4a = conv_block([256, 256, 1024], stage=4, block='a') + self.l4b = id_block([256, 256, 1024], stage=4, block='b') + self.l4c = id_block([256, 256, 1024], stage=4, block='c') + self.l4d = id_block([256, 256, 1024], stage=4, block='d') + self.l4e = id_block([256, 256, 1024], stage=4, block='e') + self.l4f = id_block([256, 256, 1024], stage=4, block='f') + + self.l5a = conv_block([512, 512, 2048], stage=5, block='a') + self.l5b = id_block([512, 512, 2048], stage=5, block='b') + self.l5c = id_block([512, 512, 2048], stage=5, block='c') + + self.avg_pool = self.track_layer( + tf.layers.AveragePooling2D( + (7, 7), strides=(7, 7), data_format=data_format)) + + if self.include_top: + self.fc1000 = self.track_layer( + tf.layers.Dense(classes, name='fc1000')) + else: + reduction_indices = [1, 2] if data_format == 'channels_last' else [2, 3] + reduction_indices = tf.constant(reduction_indices) + if pooling == 'avg': + self.global_pooling = functools.partial( + tf.reduce_mean, + reduction_indices=reduction_indices, + keep_dims=False) + elif pooling == 'max': + self.global_pooling = functools.partial( + tf.reduce_max, reduction_indices=reduction_indices, keep_dims=False) + else: + self.global_pooling = None + + def call(self, input_tensor, training=False): + x = self.conv1(input_tensor) + x = self.bn_conv1(x, training=training) + x = tf.nn.relu(x) + x = self.max_pool(x) + + x = self.l2a(x, training=training) + x = self.l2b(x, training=training) + x = self.l2c(x, training=training) + + x = self.l3a(x, training=training) + x = self.l3b(x, training=training) + x = self.l3c(x, training=training) + x = self.l3d(x, training=training) + + x = self.l4a(x, training=training) + x = self.l4b(x, training=training) + x = self.l4c(x, training=training) + x = self.l4d(x, training=training) + x = self.l4e(x, training=training) + x = self.l4f(x, training=training) + + x = self.l5a(x, training=training) + x = self.l5b(x, training=training) + x = self.l5c(x, training=training) + + x = self.avg_pool(x) + + if self.include_top: + return self.fc1000(tf.layers.flatten(x)) + elif self.global_pooling: + return self.global_pooling(x) + else: + return x diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py new file mode 100644 index 0000000000..736a75332f --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py @@ -0,0 +1,163 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests and benchmarks for ResNet50 under graph execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 +from tensorflow.contrib.summary import summary_test_util + + +def data_format(): + return 'channels_first' if tf.test.is_gpu_available() else 'channels_last' + + +def image_shape(batch_size): + if data_format() == 'channels_first': + return [batch_size, 3, 224, 224] + return [batch_size, 224, 224, 3] + + +def random_batch(batch_size): + images = np.random.rand(*image_shape(batch_size)).astype(np.float32) + num_classes = 1000 + labels = np.random.randint( + low=0, high=num_classes, size=[batch_size]).astype(np.int32) + one_hot = np.zeros((batch_size, num_classes)).astype(np.float32) + one_hot[np.arange(batch_size), labels] = 1. + return images, one_hot + + +class ResNet50GraphTest(tf.test.TestCase): + + def testApply(self): + batch_size = 64 + with tf.Graph().as_default(): + images = tf.placeholder(tf.float32, image_shape(None)) + model = resnet50.ResNet50(data_format()) + predictions = model(images) + + init = tf.global_variables_initializer() + + with tf.Session() as sess: + sess.run(init) + np_images, _ = random_batch(batch_size) + out = sess.run(predictions, feed_dict={images: np_images}) + self.assertAllEqual([64, 1000], out.shape) + + def testTrainWithSummary(self): + with tf.Graph().as_default(): + images = tf.placeholder(tf.float32, image_shape(None), name='images') + labels = tf.placeholder(tf.float32, [None, 1000], name='labels') + + tf.train.get_or_create_global_step() + logdir = tempfile.mkdtemp() + with tf.contrib.summary.always_record_summaries(): + with tf.contrib.summary.create_summary_file_writer( + logdir, max_queue=0, + name='t0').as_default(): + model = resnet50.ResNet50(data_format()) + logits = model(images, training=True) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels) + tf.contrib.summary.scalar(name='loss', tensor=loss) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + train_op = optimizer.minimize(loss) + + init = tf.global_variables_initializer() + self.assertEqual(321, len(tf.global_variables())) + + batch_size = 32 + with tf.Session() as sess: + sess.run(init) + sess.run(tf.contrib.summary.summary_writer_initializer_op()) + np_images, np_labels = random_batch(batch_size) + sess.run([train_op, tf.contrib.summary.all_summary_ops()], + feed_dict={images: np_images, labels: np_labels}) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].tag, 'loss') + + +class ResNet50Benchmarks(tf.test.Benchmark): + + def _report(self, label, start, num_iters, batch_size): + avg_time = (time.time() - start) / num_iters + dev = 'gpu' if tf.test.is_gpu_available() else 'cpu' + name = 'graph_%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format()) + extras = {'examples_per_sec': batch_size / avg_time} + self.report_benchmark( + iters=num_iters, wall_time=avg_time, name=name, extras=extras) + + def benchmark_graph_apply(self): + with tf.Graph().as_default(): + images = tf.placeholder(tf.float32, image_shape(None)) + model = resnet50.ResNet50(data_format()) + predictions = model(images) + + init = tf.global_variables_initializer() + + batch_size = 64 + with tf.Session() as sess: + sess.run(init) + np_images, _ = random_batch(batch_size) + num_burn, num_iters = (3, 30) + for _ in range(num_burn): + sess.run(predictions, feed_dict={images: np_images}) + start = time.time() + for _ in range(num_iters): + # Comparison with the eager execution benchmark in resnet50_test.py + # isn't entirely fair as the time here includes the cost of copying + # the feeds from CPU memory to GPU. + sess.run(predictions, feed_dict={images: np_images}) + self._report('apply', start, num_iters, batch_size) + + def benchmark_graph_train(self): + for batch_size in [16, 32, 64]: + with tf.Graph().as_default(): + np_images, np_labels = random_batch(batch_size) + dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat() + (images, labels) = dataset.make_one_shot_iterator().get_next() + + model = resnet50.ResNet50(data_format()) + logits = model(images, training=True) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + train_op = optimizer.minimize(loss) + + init = tf.global_variables_initializer() + with tf.Session() as sess: + sess.run(init) + (num_burn, num_iters) = (5, 10) + for _ in range(num_burn): + sess.run(train_op) + start = time.time() + for _ in range(num_iters): + sess.run(train_op) + self._report('train', start, num_iters, batch_size) + + +if __name__ == '__main__': + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py new file mode 100644 index 0000000000..d6389f2e38 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py @@ -0,0 +1,234 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests and benchmarks for the ResNet50 model, executed eagerly.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import tempfile +import time + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 +from tensorflow.contrib.summary import summary_test_util +from tensorflow.python.client import device_lib + + +def device_and_data_format(): + return ('/gpu:0', 'channels_first') if tfe.num_gpus() else ('/cpu:0', + 'channels_last') + + +def random_batch(batch_size): + _, data_format = device_and_data_format() + + shape = (3, 224, 224) if data_format == 'channels_first' else (224, 224, 3) + shape = (batch_size,) + shape + + num_classes = 1000 + images = tf.random_uniform(shape) + labels = tf.random_uniform( + [batch_size], minval=0, maxval=num_classes, dtype=tf.int32) + one_hot = tf.one_hot(labels, num_classes) + + return images, one_hot + + +def train_one_step(model, images, labels, optimizer): + + def model_loss(): + logits = model(images, training=True) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels) + tf.contrib.summary.scalar(name='loss', tensor=loss) + return loss + + optimizer.minimize(model_loss) + + +class ResNet50Test(tf.test.TestCase): + + def test_apply(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + with tf.device(device): + images, _ = random_batch(2) + output = model(images) + self.assertEqual((2, 1000), output.shape) + + def test_apply_no_top(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format, include_top=False) + with tf.device(device): + images, _ = random_batch(2) + output = model(images) + output_shape = ((2, 2048, 1, 1) + if data_format == 'channels_first' else (2, 1, 1, 2048)) + self.assertEqual(output_shape, output.shape) + + def test_apply_with_pooling(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format, include_top=False, pooling='avg') + with tf.device(device): + images, _ = random_batch(2) + output = model(images) + self.assertEqual((2, 2048), output.shape) + + def test_train(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + tf.train.get_or_create_global_step() + logdir = tempfile.mkdtemp() + with tf.contrib.summary.create_summary_file_writer( + logdir, max_queue=0, + name='t0').as_default(), tf.contrib.summary.always_record_summaries(): + with tf.device(device): + optimizer = tf.train.GradientDescentOptimizer(0.1) + images, labels = random_batch(2) + train_one_step(model, images, labels, optimizer) + self.assertEqual(320, len(model.variables)) + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].tag, 'loss') + + def test_no_garbage(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + optimizer = tf.train.GradientDescentOptimizer(0.1) + with tf.device(device): + images, labels = random_batch(2) + gc.disable() + # Warm up. Note that this first run does create significant amounts of + # garbage to be collected. The hope is that this is a build-only effect, + # and a subsequent training loop will create nothing which needs to be + # collected. + train_one_step(model, images, labels, optimizer) + gc.collect() + previous_gc_debug_flags = gc.get_debug() + gc.set_debug(gc.DEBUG_SAVEALL) + for _ in range(2): + # Run twice to ensure that garbage that is created on the first + # iteration is no longer accessible. + train_one_step(model, images, labels, optimizer) + gc.collect() + # There should be no garbage requiring collection. + self.assertEqual(0, len(gc.garbage)) + gc.set_debug(previous_gc_debug_flags) + gc.enable() + + +class MockIterator(object): + + def __init__(self, tensors): + self._tensors = [tf.identity(x) for x in tensors] + + def next(self): + return self._tensors + + +class ResNet50Benchmarks(tf.test.Benchmark): + + def _train_batch_sizes(self): + """Choose batch sizes based on GPU capability.""" + for device in device_lib.list_local_devices(): + if 'GPU:0' in device.name: + # Avoid OOM errors with larger batch sizes, which seem to cause errors + # later on even if caught. + # + # TODO(allenl): Base this on device memory; memory limit information + # during the test seems to exclude the amount TensorFlow has allocated, + # which isn't useful. + if 'K20' in device.physical_device_desc: + return (16,) + if 'P100' in device.physical_device_desc: + return (16, 32, 64) + return (16, 32) + + def _report(self, label, start, num_iters, device, batch_size, data_format): + avg_time = (time.time() - start) / num_iters + dev = 'cpu' if 'cpu' in device else 'gpu' + name = '%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format) + extras = {'examples_per_sec': batch_size / avg_time} + self.report_benchmark( + iters=num_iters, wall_time=avg_time, name=name, extras=extras) + + def _force_gpu_sync(self): + # If this function is called in the context of a GPU device + # (e.g., inside a 'with tf.device("/gpu:0")' block) + # then this will force a copy from CPU->GPU->CPU, which forces + # a sync. This is a roundabout way, yes. + tf.constant(1.).cpu() + + def benchmark_eager_apply(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + batch_size = 64 + num_burn = 5 + num_iters = 30 + with tf.device(device): + images, _ = random_batch(batch_size) + for _ in xrange(num_burn): + model(images).cpu() + gc.collect() + start = time.time() + for _ in xrange(num_iters): + model(images).cpu() + self._report('eager_apply', start, num_iters, device, batch_size, + data_format) + + def _benchmark_eager_train(self, label, make_iterator): + device, data_format = device_and_data_format() + for batch_size in self._train_batch_sizes(): + (images, labels) = random_batch(batch_size) + num_burn = 3 + num_iters = 10 + model = resnet50.ResNet50(data_format) + optimizer = tf.train.GradientDescentOptimizer(0.1) + + with tf.device(device): + iterator = make_iterator((images, labels)) + for _ in xrange(num_burn): + (images, labels) = iterator.next() + train_one_step(model, images, labels, optimizer) + self._force_gpu_sync() + gc.collect() + + start = time.time() + for _ in xrange(num_iters): + (images, labels) = iterator.next() + train_one_step(model, images, labels, optimizer) + self._force_gpu_sync() + self._report(label, start, num_iters, device, batch_size, data_format) + + def benchmark_eager_train(self): + self._benchmark_eager_train('eager_train', MockIterator) + + def benchmark_eager_train_datasets(self): + + def make_iterator(tensors): + with tf.device('/device:CPU:0'): + ds = tf.data.Dataset.from_tensors(tensors).repeat() + return tfe.Iterator(ds) + + self._benchmark_eager_train('eager_train_dataset', make_iterator) + + +if __name__ == '__main__': + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD new file mode 100644 index 0000000000..b657d31f35 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD @@ -0,0 +1,26 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "rnn_colorbot", + srcs = ["rnn_colorbot.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "@six_archive//:six", + ], +) + +cuda_py_test( + name = "rnn_colorbot_test", + srcs = ["rnn_colorbot_test.py"], + additional_deps = [ + ":rnn_colorbot", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md b/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md new file mode 100644 index 0000000000..fabd7b3e20 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md @@ -0,0 +1,26 @@ +RNN Colorbot: An RNN that predicts colors using eager execution. + +To train and generate colors, run: + +``` +python rnn_colorbot.py +``` + +This example shows how to: + 1. read, process, (one-hot) encode, and pad text data via the + Datasets API; + 2. build a trainable model; + 3. implement a multi-layer RNN using Python control flow + constructs (e.g., a for loop); + 4. train a model using an iterative gradient-based method; and + 5. log training and evaluation loss for consumption by TensorBoard + (to view summaries, use: tensorboard --log_dir=/summaries). + +The data used in this example is licensed under the Creative Commons +Attribution-ShareAlike License and is available at + https://en.wikipedia.org/wiki/List_of_colors:_A-F + https://en.wikipedia.org/wiki/List_of_colors:_G-M + https://en.wikipedia.org/wiki/List_of_colors:_N-Z + +This example was adapted from + https://github.com/random-forests/tensorflow-workshop/tree/master/extras/colorbot diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py new file mode 100644 index 0000000000..318962c634 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py @@ -0,0 +1,338 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""TensorFlow Eager Execution Example: RNN Colorbot. + +This example builds, trains, and evaluates a multi-layer RNN that can be +run with eager execution enabled. The RNN is trained to map color names to +their RGB values: it takes as input a one-hot encoded character sequence and +outputs a three-tuple (R, G, B) (scaled by 1/255). + +For example, say we'd like the RNN Colorbot to generate the RGB values for the +color white. To represent our query in a form that the Colorbot could +understand, we would create a sequence of five 256-long vectors encoding the +ASCII values of the characters in "white". The first vector in our sequence +would be 0 everywhere except for the ord("w")-th position, where it would be +1, the second vector would be 0 everywhere except for the +ord("h")-th position, where it would be 1, and similarly for the remaining three +vectors. We refer to such indicator vectors as "one-hot encodings" of +characters. After consuming these vectors, a well-trained Colorbot would output +the three tuple (1, 1, 1), since the RGB values for white are (255, 255, 255). +We are of course free to ask the colorbot to generate colors for any string we'd +like, such as "steel gray," "tensorflow orange," or "green apple," though +your mileage may vary as your queries increase in creativity. + +This example shows how to: + 1. read, process, (one-hot) encode, and pad text data via the + Datasets API; + 2. build a trainable model; + 3. implement a multi-layer RNN using Python control flow + constructs (e.g., a for loop); + 4. train a model using an iterative gradient-based method; and + +The data used in this example is licensed under the Creative Commons +Attribution-ShareAlike License and is available at + https://en.wikipedia.org/wiki/List_of_colors:_A-F + https://en.wikipedia.org/wiki/List_of_colors:_G-M + https://en.wikipedia.org/wiki/List_of_colors:_N-Z + +This example was adapted from + https://github.com/random-forests/tensorflow-workshop/tree/master/extras/colorbot +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import functools +import os +import sys +import time + +import six +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.python.eager import context + +try: + import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + + +def parse(line): + """Parse a line from the colors dataset.""" + + # Each line of the dataset is comma-separated and formatted as + # color_name, r, g, b + # so `items` is a list [color_name, r, g, b]. + items = tf.string_split([line], ",").values + rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255. + # Represent the color name as a one-hot encoded character sequence. + color_name = items[0] + chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256) + # The sequence length is needed by our RNN. + length = tf.cast(tf.shape(chars)[0], dtype=tf.int64) + return rgb, chars, length + + +def load_dataset(data_dir, url, batch_size): + """Loads the colors data at path into a PaddedDataset.""" + + # Downloads data at url into data_dir/basename(url). The dataset has a header + # row (color_name, r, g, b) followed by comma-separated lines. + path = tf.contrib.learn.datasets.base.maybe_download( + os.path.basename(url), data_dir, url) + + # This chain of commands loads our data by: + # 1. skipping the header; (.skip(1)) + # 2. parsing the subsequent lines; (.map(parse)) + # 3. shuffling the data; (.shuffle(...)) + # 3. grouping the data into padded batches (.padded_batch(...)). + dataset = tf.data.TextLineDataset(path).skip(1).map(parse).shuffle( + buffer_size=10000).padded_batch( + batch_size, padded_shapes=([None], [None, None], [])) + return dataset + + +# pylint: disable=not-callable +class RNNColorbot(tfe.Network): + """Multi-layer (LSTM) RNN that regresses on real-valued vector labels. + """ + + def __init__(self, rnn_cell_sizes, label_dimension, keep_prob): + """Constructs an RNNColorbot. + + Args: + rnn_cell_sizes: list of integers denoting the size of each LSTM cell in + the RNN; rnn_cell_sizes[i] is the size of the i-th layer cell + label_dimension: the length of the labels on which to regress + keep_prob: (1 - dropout probability); dropout is applied to the outputs of + each LSTM layer + """ + super(RNNColorbot, self).__init__(name="") + self.label_dimension = label_dimension + self.keep_prob = keep_prob + + # Note the calls to `track_layer` below; these calls register the layers as + # network components that house trainable variables. + self.cells = [ + self.track_layer(tf.nn.rnn_cell.BasicLSTMCell(size)) + for size in rnn_cell_sizes + ] + self.relu = self.track_layer( + tf.layers.Dense(label_dimension, activation=tf.nn.relu, name="relu")) + + def call(self, chars, sequence_length, training=False): + """Implements the RNN logic and prediction generation. + + Args: + chars: a Tensor of dimension [batch_size, time_steps, 256] holding a + batch of one-hot encoded color names + sequence_length: a Tensor of dimension [batch_size] holding the length + of each character sequence (i.e., color name) + training: whether the invocation is happening during training + + Returns: + A tensor of dimension [batch_size, label_dimension] that is produced by + passing chars through a multi-layer RNN and applying a ReLU to the final + hidden state. + """ + # Transpose the first and second dimensions so that chars is of shape + # [time_steps, batch_size, dimension]. + chars = tf.transpose(chars, [1, 0, 2]) + # The outer loop cycles through the layers of the RNN; the inner loop + # executes the time steps for a particular layer. + batch_size = int(chars.shape[1]) + for l in range(len(self.cells)): + cell = self.cells[l] + outputs = [] + state = cell.zero_state(batch_size, tf.float32) + # Unstack the inputs to obtain a list of batches, one for each time step. + chars = tf.unstack(chars, axis=0) + for ch in chars: + output, state = cell(ch, state) + outputs.append(output) + # The outputs of this layer are the inputs of the subsequent layer. + chars = tf.stack(outputs, axis=0) + if training: + chars = tf.nn.dropout(chars, self.keep_prob) + # Extract the correct output (i.e., hidden state) for each example. All the + # character sequences in this batch were padded to the same fixed length so + # that they could be easily fed through the above RNN loop. The + # `sequence_length` vector tells us the true lengths of the character + # sequences, letting us obtain for each sequence the hidden state that was + # generated by its non-padding characters. + batch_range = [i for i in range(batch_size)] + indices = tf.stack([sequence_length - 1, batch_range], axis=1) + hidden_states = tf.gather_nd(chars, indices) + return self.relu(hidden_states) + + +def loss(labels, predictions): + """Computes mean squared loss.""" + return tf.reduce_mean(tf.square(predictions - labels)) + + +def test(model, eval_data): + """Computes the average loss on eval_data, which should be a Dataset.""" + avg_loss = tfe.metrics.Mean("loss") + for (labels, chars, sequence_length) in tfe.Iterator(eval_data): + predictions = model(chars, sequence_length, training=False) + avg_loss(loss(labels, predictions)) + print("eval/loss: %.6f\n" % avg_loss.result()) + with tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar("loss", avg_loss.result()) + + +def train_one_epoch(model, optimizer, train_data, log_interval=10): + """Trains model on train_data using optimizer.""" + + tf.train.get_or_create_global_step() + + def model_loss(labels, chars, sequence_length): + predictions = model(chars, sequence_length, training=True) + loss_value = loss(labels, predictions) + tf.contrib.summary.scalar("loss", loss_value) + return loss_value + + for (batch, (labels, chars, sequence_length)) in enumerate( + tfe.Iterator(train_data)): + with tf.contrib.summary.record_summaries_every_n_global_steps(log_interval): + batch_model_loss = functools.partial(model_loss, labels, chars, + sequence_length) + optimizer.minimize( + batch_model_loss, global_step=tf.train.get_global_step()) + if log_interval and batch % log_interval == 0: + print("train/batch #%d\tloss: %.6f" % (batch, batch_model_loss())) + + +SOURCE_TRAIN_URL = "https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv" +SOURCE_TEST_URL = "https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv" + + +def main(_): + data_dir = os.path.join(FLAGS.dir, "data") + train_data = load_dataset( + data_dir=data_dir, url=SOURCE_TRAIN_URL, batch_size=FLAGS.batch_size) + eval_data = load_dataset( + data_dir=data_dir, url=SOURCE_TEST_URL, batch_size=FLAGS.batch_size) + + model = RNNColorbot( + rnn_cell_sizes=FLAGS.rnn_cell_sizes, + label_dimension=3, + keep_prob=FLAGS.keep_probability) + optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) + + if FLAGS.no_gpu or tfe.num_gpus() <= 0: + print(tfe.num_gpus()) + device = "/cpu:0" + else: + device = "/gpu:0" + print("Using device %s." % device) + + log_dir = os.path.join(FLAGS.dir, "summaries") + tf.gfile.MakeDirs(log_dir) + train_summary_writer = tf.contrib.summary.create_summary_file_writer( + os.path.join(log_dir, "train"), flush_secs=10) + test_summary_writer = tf.contrib.summary.create_summary_file_writer( + os.path.join(log_dir, "eval"), flush_secs=10, name="eval") + + with tf.device(device): + for epoch in range(FLAGS.num_epochs): + start = time.time() + with train_summary_writer.as_default(): + train_one_epoch(model, optimizer, train_data, FLAGS.log_interval) + end = time.time() + print("train/time for epoch #%d: %.2f" % (epoch, end - start)) + with test_summary_writer.as_default(): + test(model, eval_data) + + print("Colorbot is ready to generate colors!") + while True: + try: + color_name = six.moves.input( + "Give me a color name (or press enter to exit): ") + except EOFError: + return + + if not color_name: + return + + _, chars, length = parse(color_name) + with tf.device(device): + (chars, length) = (tf.identity(chars), tf.identity(length)) + chars = tf.expand_dims(chars, 0) + length = tf.expand_dims(length, 0) + preds = tf.unstack(model(chars, length, training=False)[0]) + + # Predictions cannot be negative, as they are generated by a ReLU layer; + # they may, however, be greater than 1. + clipped_preds = tuple(min(float(p), 1.0) for p in preds) + rgb = tuple(int(p * 255) for p in clipped_preds) + print("rgb:", rgb) + data = [[clipped_preds]] + if HAS_MATPLOTLIB: + plt.imshow(data) + plt.title(color_name) + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dir", + type=str, + default="/tmp/rnn_colorbot/", + help="Directory to download data files and save logs.") + parser.add_argument( + "--log_interval", + type=int, + default=10, + metavar="N", + help="Log training loss every log_interval batches.") + parser.add_argument( + "--num_epochs", type=int, default=20, help="Number of epochs to train.") + parser.add_argument( + "--rnn_cell_sizes", + type=int, + nargs="+", + default=[256, 128], + help="List of sizes for each layer of the RNN.") + parser.add_argument( + "--batch_size", + type=int, + default=64, + help="Batch size for training and eval.") + parser.add_argument( + "--keep_probability", + type=float, + default=0.5, + help="Keep probability for dropout between layers.") + parser.add_argument( + "--learning_rate", + type=float, + default=0.01, + help="Learning rate to be used during training.") + parser.add_argument( + "--no_gpu", + action="store_true", + default=False, + help="Disables GPU usage even if a GPU is available.") + + FLAGS, unparsed = parser.parse_known_args() + tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py new file mode 100644 index 0000000000..75b342ba78 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py @@ -0,0 +1,71 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.contrib.eager.python.examples.rnn_colorbot import rnn_colorbot + + +LABEL_DIMENSION = 5 + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" + + +def random_dataset(): + batch_size = 64 + time_steps = 10 + alphabet = 50 + chars = tf.one_hot( + tf.random_uniform( + [batch_size, time_steps], minval=0, maxval=alphabet, dtype=tf.int32), + alphabet) + sequence_length = tf.constant( + [time_steps for _ in range(batch_size)], dtype=tf.int64) + labels = tf.random_normal([batch_size, LABEL_DIMENSION]) + return tf.data.Dataset.from_tensors((labels, chars, sequence_length)) + + +class RNNColorbotTest(tf.test.TestCase): + + def testTrainOneEpoch(self): + model = rnn_colorbot.RNNColorbot( + rnn_cell_sizes=[256, 128, 64], + label_dimension=LABEL_DIMENSION, + keep_prob=1.0) + optimizer = tf.train.AdamOptimizer(learning_rate=.01) + dataset = random_dataset() + with tf.device(device()): + rnn_colorbot.train_one_epoch(model, optimizer, dataset) + + def testTest(self): + model = rnn_colorbot.RNNColorbot( + rnn_cell_sizes=[256], + label_dimension=LABEL_DIMENSION, + keep_prob=1.0) + dataset = random_dataset() + with tf.device(device()): + rnn_colorbot.test(model, dataset) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD new file mode 100644 index 0000000000..db2587bf2c --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "rnn_ptb", + srcs = ["rnn_ptb.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + ], +) + +cuda_py_test( + name = "rnn_ptb_test", + srcs = ["rnn_ptb_test.py"], + additional_deps = [ + ":rnn_ptb", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) + +cuda_py_test( + name = "rnn_ptb_graph_test", + srcs = ["rnn_ptb_graph_test.py"], + additional_deps = [ + ":rnn_ptb", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md b/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md new file mode 100644 index 0000000000..ea92d59e58 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md @@ -0,0 +1,42 @@ +Recurrent Neural Network model. + +Implements a language modeling network described in +https://www.tensorflow.org/tutorials/recurrent +that is compatible with (and idiomatic for) eager execution. + +To run: + +- Download and extract the Penn Treebank dataset from + http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz + + ```sh + tar xvzf simple-examples.tgz -C /tmp + ``` + +- Run: `python rnn_ptb.py --data-dir=/tmp/simple-examples/data` + + +Benchmarks (using synthetic data): + +``` +# Using eager execution +bazel run -c opt --config=cuda :rnn_ptb_test -- --benchmarks=. + +# Using graph execution +bazel run -c opt --config=cuda :rnn_ptb_graph_test -- --benchmarks=. +``` + +(Or remove the `--config=cuda` flag for running on CPU instead of GPU). + +On October 31, 2017, the benchmarks demostrated slightly better performance +(3-6%) for graph execution over eager execution for this particular model when +using a single NVIDIA Titan X (Pascal) GPU on a host with an Intel Xeon E5-1650 +CPU @ 3.50GHz and a batch size of 32. + +| Benchmark name | examples/second | +| ------------------------------------ | --------------- | +| eager_cudnn_train_large_gpu_batch_20 | 938 | +| graph_cudnn_train_large_gpu_batch_20 | 971 | +| eager_cudnn_train_small_gpu_batch_20 | 2433 | +| graph_cudnn_train_small_gpu_batch_20 | 2585 | + diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py new file mode 100644 index 0000000000..c67d77b386 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py @@ -0,0 +1,348 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Penn Treebank RNN model definition compatible with eager execution. + +Model similar to +https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb + +Usage: python ./rnn_ptb.py --data-path= + +Penn Treebank (PTB) dataset from: +http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz +""" +import argparse +import os +import sys +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn +from tensorflow.contrib.eager.python import tfe + + +class RNN(tfe.Network): + """A static RNN. + + Similar to tf.nn.static_rnn, implemented as a tf.layer.Layer. + """ + + def __init__(self, hidden_dim, num_layers, keep_ratio): + super(RNN, self).__init__() + self.keep_ratio = keep_ratio + for _ in range(num_layers): + self.track_layer(tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_dim)) + + def call(self, input_seq, training): + batch_size = int(input_seq.shape[1]) + for c in self.layers: + state = c.zero_state(batch_size, tf.float32) + outputs = [] + input_seq = tf.unstack(input_seq, num=int(input_seq.shape[0]), axis=0) + for inp in input_seq: + output, state = c(inp, state) + outputs.append(output) + + input_seq = tf.stack(outputs, axis=0) + if training: + input_seq = tf.nn.dropout(input_seq, self.keep_ratio) + return input_seq, None + + +class Embedding(tf.layers.Layer): + """An Embedding layer.""" + + def __init__(self, vocab_size, embedding_dim, **kwargs): + super(Embedding, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.embedding_dim = embedding_dim + + def build(self, _): + self.embedding = self.add_variable( + "embedding_kernel", + shape=[self.vocab_size, self.embedding_dim], + dtype=tf.float32, + initializer=tf.random_uniform_initializer(-0.1, 0.1), + trainable=True) + + def call(self, x): + return tf.nn.embedding_lookup(self.embedding, x) + + +class PTBModel(tfe.Network): + """LSTM for word language modelling. + + Model described in: + (Zaremba, et. al.) Recurrent Neural Network Regularization + http://arxiv.org/abs/1409.2329 + + See also: + https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb + """ + + def __init__(self, + vocab_size, + embedding_dim, + hidden_dim, + num_layers, + dropout_ratio, + use_cudnn_rnn=True): + super(PTBModel, self).__init__() + + self.keep_ratio = 1 - dropout_ratio + self.use_cudnn_rnn = use_cudnn_rnn + self.embedding = self.track_layer(Embedding(vocab_size, embedding_dim)) + + if self.use_cudnn_rnn: + self.rnn = cudnn_rnn.CudnnLSTM( + num_layers, hidden_dim, dropout=dropout_ratio) + else: + self.rnn = RNN(hidden_dim, num_layers, self.keep_ratio) + self.track_layer(self.rnn) + + self.linear = self.track_layer( + tf.layers.Dense( + vocab_size, + kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))) + self._output_shape = [-1, embedding_dim] + + def call(self, input_seq, training): + """Run the forward pass of PTBModel. + + Args: + input_seq: [length, batch] shape int64 tensor. + training: Is this a training call. + Returns: + outputs tensors of inference. + """ + y = self.embedding(input_seq) + if training: + y = tf.nn.dropout(y, self.keep_ratio) + y, _ = self.rnn(y, training=training) + return self.linear(tf.reshape(y, self._output_shape)) + + +def clip_gradients(grads_and_vars, clip_ratio): + gradients, variables = zip(*grads_and_vars) + clipped, _ = tf.clip_by_global_norm(gradients, clip_ratio) + return zip(clipped, variables) + + +def loss_fn(model, inputs, targets, training): + labels = tf.reshape(targets, [-1]) + outputs = model(inputs, training) + return tf.reduce_mean( + tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=labels, logits=outputs)) + + +def _divide_into_batches(data, batch_size): + """Convert a sequence to a batch of sequences.""" + nbatch = data.shape[0] // batch_size + data = data[:nbatch * batch_size] + data = data.reshape(batch_size, -1).transpose() + return data + + +def _get_batch(data, i, seq_len): + slen = min(seq_len, data.shape[0] - 1 - i) + inputs = data[i:i + slen, :] + target = data[i + 1:i + 1 + slen, :] + return tf.constant(inputs), tf.constant(target) + + +def evaluate(model, data): + """evaluate an epoch.""" + total_loss = 0.0 + total_batches = 0 + start = time.time() + for _, i in enumerate(range(0, data.shape[0] - 1, FLAGS.seq_len)): + inp, target = _get_batch(data, i, FLAGS.seq_len) + loss = loss_fn(model, inp, target, training=False) + total_loss += loss.numpy() + total_batches += 1 + time_in_ms = (time.time() - start) * 1000 + sys.stderr.write("eval loss %.2f (eval took %d ms)\n" % + (total_loss / total_batches, time_in_ms)) + return total_loss + + +def train(model, optimizer, train_data, sequence_length, clip_ratio): + """training an epoch.""" + + def model_loss(inputs, targets): + return loss_fn(model, inputs, targets, training=True) + + grads = tfe.implicit_gradients(model_loss) + + total_time = 0 + for batch, i in enumerate(range(0, train_data.shape[0] - 1, sequence_length)): + train_seq, train_target = _get_batch(train_data, i, sequence_length) + start = time.time() + optimizer.apply_gradients( + clip_gradients(grads(train_seq, train_target), clip_ratio)) + total_time += (time.time() - start) + if batch % 10 == 0: + time_in_ms = (total_time * 1000) / (batch + 1) + sys.stderr.write("batch %d: training loss %.2f, avg step time %d ms\n" % + (batch, model_loss(train_seq, train_target).numpy(), + time_in_ms)) + + +class Datasets(object): + """Processed form of the Penn Treebank dataset.""" + + def __init__(self, path): + """Load the Penn Treebank dataset. + + Args: + path: Path to the data/ directory of the dataset from from Tomas Mikolov's + webpage - http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz + """ + + self.word2idx = {} # string -> integer id + self.idx2word = [] # integer id -> word string + # Files represented as a list of integer ids (as opposed to list of string + # words). + self.train = self.tokenize(os.path.join(path, "ptb.train.txt")) + self.valid = self.tokenize(os.path.join(path, "ptb.valid.txt")) + + def vocab_size(self): + return len(self.idx2word) + + def add(self, word): + if word not in self.word2idx: + self.idx2word.append(word) + self.word2idx[word] = len(self.idx2word) - 1 + + def tokenize(self, path): + """Read text file in path and return a list of integer token ids.""" + tokens = 0 + with tf.gfile.Open(path, "r") as f: + for line in f: + words = line.split() + [""] + tokens += len(words) + for word in words: + self.add(word) + + # Tokenize file content + with tf.gfile.Open(path, "r") as f: + ids = np.zeros(tokens).astype(np.int64) + token = 0 + for line in f: + words = line.split() + [""] + for word in words: + ids[token] = self.word2idx[word] + token += 1 + + return ids + + +def small_model(use_cudnn_rnn): + """Returns a PTBModel with a 'small' configuration.""" + return PTBModel( + vocab_size=10000, + embedding_dim=200, + hidden_dim=200, + num_layers=2, + dropout_ratio=0., + use_cudnn_rnn=use_cudnn_rnn) + + +def large_model(use_cudnn_rnn): + """Returns a PTBModel with a 'large' configuration.""" + return PTBModel( + vocab_size=10000, + embedding_dim=650, + hidden_dim=650, + num_layers=2, + dropout_ratio=0.5, + use_cudnn_rnn=use_cudnn_rnn) + + +def main(_): + tfe.enable_eager_execution() + + if not FLAGS.data_path: + raise ValueError("Must specify --data-path") + corpus = Datasets(FLAGS.data_path) + train_data = _divide_into_batches(corpus.train, FLAGS.batch_size) + eval_data = _divide_into_batches(corpus.valid, 10) + + have_gpu = tfe.num_gpus() > 0 + use_cudnn_rnn = not FLAGS.no_use_cudnn_rnn and have_gpu + + with tfe.restore_variables_on_create( + tf.train.latest_checkpoint(FLAGS.logdir)): + with tf.device("/device:GPU:0" if have_gpu else None): + # Make learning_rate a Variable so it can be included in the checkpoint + # and we can resume training with the last saved learning_rate. + learning_rate = tfe.Variable(20.0, name="learning_rate") + sys.stderr.write("learning_rate=%f\n" % learning_rate.numpy()) + model = PTBModel(corpus.vocab_size(), FLAGS.embedding_dim, + FLAGS.hidden_dim, FLAGS.num_layers, FLAGS.dropout, + use_cudnn_rnn) + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + + best_loss = None + for _ in range(FLAGS.epoch): + train(model, optimizer, train_data, FLAGS.seq_len, FLAGS.clip) + eval_loss = evaluate(model, eval_data) + if not best_loss or eval_loss < best_loss: + if FLAGS.logdir: + tfe.Saver(model.trainable_weights + [learning_rate]).save( + os.path.join(FLAGS.logdir, "ckpt")) + best_loss = eval_loss + else: + learning_rate.assign(learning_rate / 4.0) + sys.stderr.write("eval_loss did not reduce in this epoch, " + "changing learning rate to %f for the next epoch\n" % + learning_rate.numpy()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--data-path", + type=str, + default="", + help="Data directory of the Penn Treebank dataset from " + "http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz") + parser.add_argument( + "--logdir", type=str, default="", help="Directory for checkpoint.") + parser.add_argument( + "--epoch", type=int, default=20, help="Number of epoches.") + parser.add_argument("--batch-size", type=int, default=20, help="Batch size.") + parser.add_argument( + "--seq-len", type=int, default=35, help="Sequence length.") + parser.add_argument( + "--embedding-dim", type=int, default=200, help="Embedding dimension.") + parser.add_argument( + "--hidden-dim", type=int, default=200, help="Hidden layer dimension.") + parser.add_argument( + "--num-layers", type=int, default=2, help="Number of RNN layers.") + parser.add_argument( + "--dropout", type=float, default=0.2, help="Drop out ratio.") + parser.add_argument( + "--clip", type=float, default=0.25, help="Gradient clipping ratio.") + parser.add_argument( + "--no-use-cudnn-rnn", + action="store_true", + default=False, + help="Disable the fast CuDNN RNN (when no gpu)") + + FLAGS, unparsed = parser.parse_known_args() + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py new file mode 100644 index 0000000000..168b5c5356 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py @@ -0,0 +1,164 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for PTBModel used for graph construction.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python.examples.rnn_ptb import rnn_ptb + + +class PTBTest(tf.test.TestCase): + + def testTrain(self): + batch_size = 20 + sequence_length = 35 + with tf.Graph().as_default(), tf.device(tf.test.gpu_device_name()): + inputs_ph = tf.placeholder(tf.int64, [sequence_length, batch_size], + "inputs") + labels_ph = tf.placeholder(tf.int64, [sequence_length, batch_size], + "labels") + + inputs = np.ones(inputs_ph.shape.as_list(), dtype=np.int64) + labels = np.ones(labels_ph.shape.as_list(), dtype=np.int64) + + model = rnn_ptb.small_model(tf.test.is_gpu_available()) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + loss = rnn_ptb.loss_fn(model, inputs_ph, labels_ph, training=True) + grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) + train_op = optimizer.apply_gradients(grads) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + sess.run(train_op, feed_dict={inputs_ph: inputs, labels_ph: labels}) + sess.run( + [train_op, loss], feed_dict={ + inputs_ph: inputs, + labels_ph: labels + }) + + +class PTBBenchmark(tf.test.Benchmark): + + BATCH_SIZE = 20 + SEQ_LEN = 35 + + def _report(self, label, start, num_iters, device, batch_size): + wall_time = (time.time() - start) / num_iters + dev = "cpu" if "cpu" in device.lower() else "gpu" + name = "%s_%s_batch_%d" % (label, dev, batch_size) + examples_per_sec = batch_size / wall_time + self.report_benchmark( + iters=num_iters, + wall_time=wall_time, + name=name, + extras={ + "examples_per_sec": examples_per_sec + }) + + def _benchmark_apply(self, label, model): + num_iters = 100 + num_warmup = 10 + dataset = tf.data.Dataset.from_tensors( + tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], + dtype=tf.int64)).repeat(num_iters + num_warmup) + inputs = dataset.make_one_shot_iterator().get_next() + + with tf.device(tf.test.gpu_device_name()): + outputs = model(inputs, training=True) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for _ in range(num_warmup): + sess.run(outputs) + gc.collect() + + start = time.time() + for _ in range(num_iters): + sess.run(outputs) + self._report(label, start, num_iters, + tf.test.gpu_device_name(), PTBBenchmark.BATCH_SIZE) + + def benchmark_apply_small(self): + self._benchmark_apply("graph_apply_small", rnn_ptb.small_model(False)) + + def benchmark_apply_large(self): + self._benchmark_apply("graph_apply_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_apply_small(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_apply("graph_cudnn_apply_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_apply_large(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_apply("graph_cudnn_apply_large", rnn_ptb.large_model(True)) + + def _benchmark_train(self, label, model): + num_iters = 100 + num_warmup = 10 + dataset = tf.data.Dataset.from_tensors( + tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], + dtype=tf.int64)).repeat(num_iters + num_warmup) + # inputs and labels have the same shape + dataset = tf.data.Dataset.zip((dataset, dataset)) + (inputs, labels) = dataset.make_one_shot_iterator().get_next() + + with tf.device(tf.test.gpu_device_name()): + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + loss = rnn_ptb.loss_fn(model, inputs, labels, training=True) + grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) + train_op = optimizer.apply_gradients(grads) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for _ in range(num_warmup): + sess.run(train_op) + gc.collect() + start = time.time() + for _ in range(num_iters): + sess.run(train_op) + self._report(label, start, num_iters, + tf.test.gpu_device_name(), PTBBenchmark.BATCH_SIZE) + + def benchmark_train_small(self): + self._benchmark_train("graph_train_small", rnn_ptb.small_model(False)) + + def benchmark_train_large(self): + self._benchmark_train("graph_train_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_train_small(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_train("graph_cudnn_train_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_train_large(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_train("graph_cudnn_train_large", rnn_ptb.large_model(True)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py new file mode 100644 index 0000000000..6f296c2aba --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py @@ -0,0 +1,154 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for PTBModel with eager execution enabled.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.contrib.eager.python.examples.rnn_ptb import rnn_ptb + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" + + +class PTBTest(tf.test.TestCase): + + def testTrain(self): + model = rnn_ptb.small_model(tfe.num_gpus() > 0) + sequence_length = 35 + data = np.ones([4 * sequence_length, 20], dtype=np.int64) + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(1.0) + # Train two epochs + rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) + rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) + + def testApply(self): + model = rnn_ptb.small_model(tfe.num_gpus() > 0) + with tf.device(device()): + model(tf.ones([35, 20], dtype=tf.int64), training=False) + + +def force_gpu_sync(): + if tfe.num_gpus(): + tf.constant(1).gpu().cpu() + + +class PTBBenchmark(tf.test.Benchmark): + + BATCH_SIZE = 20 + SEQ_LEN = 35 + + def _report(self, label, start, num_iters, dev, batch_size): + wall_time = (time.time() - start) / num_iters + dev = "cpu" if "cpu" in dev.lower() else "gpu" + name = "%s_%s_batch_%d" % (label, dev, batch_size) + examples_per_sec = batch_size / wall_time + self.report_benchmark( + iters=num_iters, + wall_time=wall_time, + name=name, + extras={ + "examples_per_sec": examples_per_sec + }) + + def _benchmark_apply(self, label, model): + with tf.device(device()): + sequence_batch = tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64) + + for _ in range(10): # Warmup + model(sequence_batch, training=False).cpu() + gc.collect() + + start = time.time() + iters = 100 + for _ in range(iters): + model(sequence_batch, training=False).cpu() + self._report(label, start, iters, device(), int(sequence_batch.shape[1])) + + def benchmark_apply_small(self): + self._benchmark_apply("eager_apply_small", rnn_ptb.small_model(False)) + + def benchmark_apply_large(self): + self._benchmark_apply("eager_apply_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_apply_small(self): + if not tfe.num_gpus(): + return + self._benchmark_apply("eager_cudnn_apply_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_apply_large(self): + if not tfe.num_gpus(): + return + self._benchmark_apply("eager_cudnn_apply_large", rnn_ptb.large_model(True)) + + def _benchmark_train(self, label, model): + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(1.) + + def model_loss(inputs, targets): + return rnn_ptb.loss_fn(model, inputs, targets, training=True) + + grads = tfe.implicit_gradients(model_loss) + + sequence_batch = tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64) + + def step(): + optimizer.apply_gradients( + rnn_ptb.clip_gradients(grads(sequence_batch, sequence_batch), 0.25)) + + for _ in range(10): # Warmup + step() + force_gpu_sync() + gc.collect() + + start = time.time() + iters = 100 + for _ in range(iters): + step() + force_gpu_sync() + self._report(label, start, iters, device(), int(sequence_batch.shape[1])) + + def benchmark_train_small(self): + self._benchmark_train("eager_train_small", rnn_ptb.small_model(False)) + + def benchmark_train_large(self): + self._benchmark_train("eager_train_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_train_small(self): + if not tfe.num_gpus(): + return + self._benchmark_train("eager_cudnn_train_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_train_large(self): + if not tfe.num_gpus(): + return + self._benchmark_train("eager_cudnn_train_large", rnn_ptb.large_model(True)) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md new file mode 100644 index 0000000000..e945bc20f4 --- /dev/null +++ b/tensorflow/contrib/eager/python/g3doc/guide.md @@ -0,0 +1,899 @@ +# TensorFlow Eager Execution + +## What is this? + +Eager execution is a feature that makes TensorFlow execute operations +immediately: concrete values are returned, instead of a computational graph to +be executed later. + +As a result, enabling eager execution provides: + +- A [NumPy](http://www.numpy.org/)-like library for numerical computation with + support for GPU acceleration and automatic differentiation. +- A flexible platform for machine learning research and experimentation. + +Eager execution is under active development. This guide walks through an +alpha/preview release. In particular, not all TensorFlow APIs currently work +with eager execution enabled, and some models may be slow to execute, compared +to models defined without using eager execution. + +## Installation + +Eager execution is **not** included in the latest release (version 1.4) of +TensorFlow. To use it, you will need to [build TensorFlow from +source](https://www.tensorflow.org/install/install_sources) or install the +nightly builds. + +For example, the nightly builds can be installed using `pip`: + +- `pip install tf-nightly` (for CPU-only TensorFlow) +- `pip install tf-nightly-gpu` (for GPU-enabled TensorFlow) + +Or using `docker`, with [Jupyter Notebook](http://jupyter.org/) support: + +```sh +# For CPU-only TensorFlow +docker pull tensorflow/tensorflow:nightly +docker run -it -p 8888:8888 tensorflow/tensorflow:nightly + +# For GPU-enabled TensorFlow: +# (Requires https://github.com/NVIDIA/nvidia-docker) +nvidia-docker pull tensorflow/tensorflow:nightly-gpu +nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu +``` + +## Getting Started + +With TensorFlow installed, eager execution is enabled via a single call: + +```python +import tensorflow as tf + +import tensorflow.contrib.eager as tfe + +tfe.enable_eager_execution() +``` + +Enabling eager execution changes how TensorFlow functions behave (in particular, +`Tensor` objects will reference concrete values instead of being symbolic +handles to nodes in a computational graph). As a result, eager execution should +be enabled at the beginning of a program and cannot be disabled afterwards in +the same program. + +Code examples in the rest of this guide assume that eager execution has been +enabled. + +## A library for numerical computation + +A significant fraction of the [TensorFlow +API](https://www.tensorflow.org/api_docs/python/) consists of numerical +operations: +[arithmetic operations](https://www.tensorflow.org/api_docs/python/tf/matmul), +[matrix operations](https://www.tensorflow.org/api_docs/python/tf/matmul), +[linear algebra operations](https://www.tensorflow.org/api_docs/python/tf/linalg), +etc. + +With eager execution enabled, these operations consume and return +multi-dimensional arrays as `Tensor` objects, similar to NumPy +[`ndarray`s](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ndarray.html). +For example: + +```python +# Multiply two 2x2 matrices +x = tf.matmul([[1, 2], + [3, 4]], + [[4, 5], + [6, 7]]) +# Add one to each element +# (tf.add supports broadcasting) +y = tf.add(x, 1) + +# Create a random random 5x3 matrix +z = tf.random_uniform([5, 3]) + +print(x) +print(y) +print(z) +``` + +Output: + +``` +tf.Tensor( +[[16 19] + [36 43]], shape=(2, 2), dtype=int32) +tf.Tensor( +[[17 20] + [37 44]], shape=(2, 2), dtype=int32) +tf.Tensor( +[[ 0.25058532 0.0929395 0.54113817] + [ 0.3108716 0.93350542 0.84909797] + [ 0.53081679 0.12788558 0.01767385] + [ 0.29725885 0.33540785 0.83588314] + [ 0.38877153 0.39720535 0.78914213]], shape=(5, 3), dtype=float32) +``` + +For convenience, these operations can also be triggered via operator overloading +of the `Tensor` object. For example, the `+` operator is equivalent to `tf.add`, +`-` to `tf.subtract`, `*` to `tf.multiply`, etc.: + +```python +x = (tf.ones([1], dtype=tf.float32) + 1) * 2 - 1 +print(x) +``` + +Output: + +``` +tf.Tensor([ 3.], shape=(1,), dtype=float32) +``` + +### Converting to and from NumPy + +The operations above automatically convert Python objects (like lists of +numbers) and NumPy arrays to `Tensor` objects. `Tensor` objects can also be used +as NumPy arrays by numpy operations. + +```python +import numpy as np + +x = tf.add(1, 1) # tf.Tensor with a value of 2 +y = tf.add(np.array(1), np.array(1)) # tf.Tensor with a value of 2 +z = np.multiply(x, y) # numpy.int64 with a value of 4 +``` + +Alternatively, they can be explicitly converted using +[`tf.constant`](https://www.tensorflow.org/api_docs/python/tf/constant), as +shown in the next example. + +Conversely, you can call the `numpy()` method of a `Tensor` object' to obtain +its NumPy `ndarray` value. For example: + +```python +import numpy as np + +np_x = np.array(2., dtype=np.float32) +x = tf.constant(np_x) + +py_y = 3. +y = tf.constant(py_y) + +z = x + y + 1 + +print(z) +print(z.numpy()) +``` + +Output: + +``` +tf.Tensor(6.0, shape=(), dtype=float32) +6.0 +``` + +### GPU acceleration + +Many TensorFlow operations support GPU acceleration. With eager execution +enabled, [computation is *not* automatically +offloaded](https://www.tensorflow.org/tutorials/using_gpu) to GPUs. Instead, you +must explicitly specify when GPUs should be used. + +The simplest way to do this is to enclose your computation in a `with +tf.device('/gpu:0')` block. Also of interest is the `tfe.num_gpus()` function, +which returns the number of available GPUs. + +For example, consider this snippet to measure the time to multiply two 1000x1000 +matrices on CPU: + +```python +import time + +def measure(x): + # The very first time a GPU is used by TensorFlow, it is initialized. + # So exclude the first run from timing. + tf.matmul(x, x) + + start = time.time() + for i in range(10): + tf.matmul(x, x) + end = time.time() + + return "Took %s seconds to multiply a %s matrix by itself 10 times" % (end - start, x.shape) + +# Run on CPU: +with tf.device("/cpu:0"): + print("CPU: %s" % measure(tf.random_normal([1000, 1000]))) + +# If a GPU is available, run on GPU: +if tfe.num_gpus() > 0: + with tf.device("/gpu:0"): + print("GPU: %s" % measure(tf.random_normal([1000, 1000]))) +``` + +Output (exact numbers will depend on the characteristics of the hardware): + +```python +CPU: Took 0.145531892776 seconds to multiply a (1000, 1000) matrix by itself 10 times +GPU: Took 0.000458955764771 seconds to multiply a (1000, 1000) matrix by itself 10 times +``` + +Alternatively, methods on the `Tensor` object can be used to explicitly copy the +`Tensor` to a different device. Operations are typically executed on the device +on which the inputs are placed. For example: + +```python +x = tf.random_normal([10, 10]) + +x_gpu0 = x.gpu() +x_cpu = x.cpu() + +_ = tf.matmul(x_cpu, x_cpu) # Runs on CPU +_ = tf.matmul(x_gpu0, x_gpu0) # Runs on GPU:0 + +if tfe.num_gpus() > 1: + x_gpu1 = x.gpu(1) + _ = tf.matmul(x_gpu1, x_gpu1) # Runs on GPU:1 +``` + +### Automatic Differentiation + +[Automatic +differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) is +very useful when implementing many machine learning algorithms (e.g., +[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training +neural networks). For this purpose, TensorFlow eager execution provides an +[autograd](https://github.com/HIPS/autograd)-style API for automatic +differentiation. Specifically, the functions: + +- `tfe.gradients_function(f)`: Returns a Python function that computes the + derivatives of the Python function `f` with respect to its arguments. `f` + must return a scalar value. When the returned function is invoked, it + returns a list of `Tensor` objects (one element for each argument of `f`). +- `tfe.value_and_gradients_function(f)`: Similar to `tfe.gradients_function`, + except that when the returned function is invoked, it returns the value of + `f` in addition to the list of derivatives of `f` with respect to its + arguments. + +These functions naturally apply to higher order differentiation as well. For +example: + +```python +def f(x): + return tf.multiply(x, x) # Or x * x +assert 9 == f(3.).numpy() + +df = tfe.gradients_function(f) +assert 6 == df(3.)[0].numpy() + +# Second order deriviative. +d2f = tfe.gradients_function(lambda x: df(x)[0]) +assert 2 == d2f(3.)[0].numpy() + +# Third order derivative. +d3f = tfe.gradients_function(lambda x : d2f(x)[0]) +assert 0 == d3f(3.)[0].numpy() +``` + +These functions can be used to train models. For example, consider the following +simple linear regression model: + +```python +def prediction(input, weight, bias): + return input * weight + bias + +# A toy dataset of points around 3 * x + 2 +NUM_EXAMPLES = 1000 +training_inputs = tf.random_normal([NUM_EXAMPLES]) +noise = tf.random_normal([NUM_EXAMPLES]) +training_outputs = training_inputs * 3 + 2 + noise + +# A loss function: Mean-squared error +def loss(weight, bias): + error = prediction(training_inputs, weight, bias) - training_outputs + return tf.reduce_mean(tf.square(error)) + +# Function that returns the the derivative of loss with respect to +# weight and bias +grad = tfe.gradients_function(loss) + +# Train for 200 steps (starting from some random choice for W and B, on the same +# batch of data). +W = 5. +B = 10. +learning_rate = 0.01 +print("Initial loss: %f" % loss(W, B).numpy()) +for i in range(200): + (dW, dB) = grad(W, B) + W -= dW * learning_rate + B -= dB * learning_rate + if i % 20 == 0: + print("Loss at step %d: %f" % (i, loss(W, B).numpy())) +print("Final loss: %f" % loss(W, B).numpy()) +print("W, B = %f, %f" % (W.numpy(), B.numpy())) +``` + +Output: (the exact numbers may vary depending on the randomness in noise) + +``` +Initial loss: 66.730003 +Loss at step 0: 64.200096 +Loss at step 20: 29.872814 +Loss at step 40: 14.233772 +Loss at step 60: 7.090570 +Loss at step 80: 3.819887 +Loss at step 100: 2.318821 +Loss at step 120: 1.628385 +Loss at step 140: 1.310142 +Loss at step 160: 1.163167 +Loss at step 180: 1.095162 +Final loss: 1.064711 +W, B = 3.094944, 2.161383 +``` + +To utilize the GPU, place the code above within a `with tf.device("/gpu:0"):` +block. (However, this particular model, with only two floating point parameters, +is unlikely to benefit from GPU acceleration.) + +### Customizing gradients + +One may want to define custom gradients for an operation, or for a function. +This may be useful for multiple reasons, including providing a more efficient +or more [numerically stable](https://en.wikipedia.org/wiki/Numerical_stability) +gradient for a sequence of operations. + +For example, consider the function `log(1 + e^x)`, which commonly occurs in the +computation of cross entropy and log likelihoods. + +```python +def log1pexp(x): +  return tf.log(1 + tf.exp(x)) +grad_log1pexp = tfe.gradients_function(log1pexp) + +# Works fine at x = 0. +assert 0.5 == float(grad_log1pexp(0.)[0]) + +# Returns a `nan` at x = 100 due to numerical instability. +import math +assert math.isnan(float(grad_log1pexp(100.)[0])) +``` + +We can define a custom gradient for the above function that analytically +simplifies the gradient expression. + +```python +@tfe.custom_gradient +def log1pexp(x): +  e = tf.exp(x) +  def grad(dy): +    return dy * (1 - 1 / (1 + e)) +  return tf.log(1 + e), grad +grad_log1pexp = tfe.gradients_function(log1pexp) + +# Works as before at x = 0. +assert 0.5 == float(grad_log1pexp(0.)[0]) + +# But now works at x = 100 as well. +assert 1.0 == float(grad_log1pexp(100.)[0]) +``` +Also notice how the gradient function implementation reuses an expression +(`tf.exp(x)`) computed during the forward pass, hence making the gradient +computation more efficient by avoiding redundant computation. + +## Building and training models + +In practice, your computation may have many parameters to be optimized (by +computing derivatives). Encapsulating them into re-usable classes/objects +makes the code easier to follow than writing a single top-level function with +many arguments. + +In fact, eager execution encourages use of the [Keras](https://keras.io)-style +"Layer" classes in the +[`tf.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/layers) +module. + +Furthermore, you may want to apply more sophisticated techniques to compute +parameter updates, such as those in +[`tf.train.Optimizer`](https://www.tensorflow.org/api_guides/python/train#Optimizers) +implementations. + +This next section walks through using the same `Optimizer` and `Layer` APIs used +to build trainable TensorFlow graphs in an environment where eager execution is +enabled. + +### Variables and Optimizers + +`tfe.Variable` objects store mutable `Tensor` values that can be accessed during +training, making automatic differentiation easier. In particular, parameters of +a model can be encapsulated in Python classes as variables. + +`tfe.gradients_function(f)` introduced earlier computes the derivatives of `f` +with respect to its arguments. However, it requires all parameters of interest +to be arguments of `f`, which becomes cumbersome when `f` depends on a large +number of trainable parameters. + +`tfe.implicit_gradients` is an alternative function with some useful properties: + +- It computes the derivatives of `f` with respect to all the `tfe.Variable`s + used by `f`. +- When the returned function is invoked, it returns a list of + (gradient value, Variable object) tuples. + +Representing model parameters as `Variable` objects, along with the use of +`tfe.implicit_gradients`, typically results in better encapsulation. For +example, the linear regression model described above can be written into a +class: + +```python +class Model(object): + def __init__(self): + self.W = tfe.Variable(5., name='weight') + self.B = tfe.Variable(10., name='bias') + + def predict(self, inputs): + return inputs * self.W + self.B + + +# The loss function to be optimized +def loss(model, inputs, targets): + error = model.predict(inputs) - targets + return tf.reduce_mean(tf.square(error)) + +# A toy dataset of points around 3 * x + 2 +NUM_EXAMPLES = 1000 +training_inputs = tf.random_normal([NUM_EXAMPLES]) +noise = tf.random_normal([NUM_EXAMPLES]) +training_outputs = training_inputs * 3 + 2 + noise + +# Define: +# 1. A model +# 2. Derivatives of a loss function with respect to model parameters +# 3. A strategy for updating the variables based on the derivatives +model = Model() +grad = tfe.implicit_gradients(loss) +optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + +# The training loop +print("Initial loss: %f" % + loss(model, training_inputs, training_outputs).numpy()) +for i in range(201): + optimizer.apply_gradients(grad(model, training_inputs, training_outputs)) + if i % 20 == 0: + print("Loss at step %d: %f" % + (i, loss(model, training_inputs, training_outputs).numpy())) +print("Final loss: %f" % loss(model, training_inputs, training_outputs).numpy()) +print("W, B = %s, %s" % (model.W.numpy(), model.B.numpy())) +``` + +Output: + +``` +Initial loss: 69.693184 +Loss at step 0: 66.987854 +Loss at step 20: 30.553387 +Loss at step 40: 14.250237 +Loss at step 60: 6.955020 +Loss at step 80: 3.690550 +Loss at step 100: 2.229739 +Loss at step 120: 1.576032 +Loss at step 140: 1.283496 +Loss at step 160: 1.152584 +Loss at step 180: 1.093999 +Final loss: 1.067780 +W, B = 3.0114281, 2.0865183 +``` + +Using `implicit_gradients` avoids the need to provide all the trainable +parameters of the model as arguments to the `loss` function. + +### Using Keras and the Layers API + +[Keras](https://keras.io) is a popular API for defining model structures. The +[`tf.keras.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/layers) +module provides a set of building blocks for models and is implemented using the +`tf.layers.Layer` subclasses in the +[`tf.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/layers) +module. We encourage the use of these same building blocks when using +TensorFlow's eager execution feature. For example, the very same linear +regression model can be built using `tf.layers.Dense`: + +```python +class Model(object): + def __init__(self): + self.layer = tf.layers.Dense(1) + + def predict(self, inputs): + return self.layer(inputs) +``` + +The `tf.layers` API makes it more convenient to define more sophisticated +models. For example, the following will train an MNIST model: + +```python +class MNISTModel(object): + def __init__(self, data_format): + # 'channels_first' is typically faster on GPUs + # while 'channels_last' is typically faster on CPUs. + # See: https://www.tensorflow.org/performance/performance_guide#data_formats + if data_format == 'channels_first': + self._input_shape = [-1, 1, 28, 28] + else: + self._input_shape = [-1, 28, 28, 1] + self.conv1 = tf.layers.Conv2D(32, 5, + padding='same', + activation=tf.nn.relu, + data_format=data_format) + self.max_pool2d = tf.layers.MaxPooling2D( + (2, 2), (2, 2), padding='same', data_format=data_format) + self.conv2 = tf.layers.Conv2D(64, 5, + padding='same', + activation=tf.nn.relu, + data_format=data_format) + self.dense1 = tf.layers.Dense(1024, activation=tf.nn.relu) + self.dropout = tf.layers.Dropout(0.5) + self.dense2 = tf.layers.Dense(10) + + def predict(self, inputs): + x = tf.reshape(inputs, self._input_shape) + x = self.max_pool2d(self.conv1(x)) + x = self.max_pool2d(self.conv2(x)) + x = tf.layers.flatten(x) + x = self.dropout(self.dense1(x)) + return self.dense2(x) + +def loss(model, inputs, targets): + return tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + logits=model.predict(inputs), labels=targets)) + + +# Load the training and validation data +from tensorflow.examples.tutorials.mnist import input_data +data = input_data.read_data_sets("./mnist_data", one_hot=True) + +# Train +device = "gpu:0" if tfe.num_gpus() else "cpu:0" +model = MNISTModel('channels_first' if tfe.num_gpus() else 'channels_last') +optimizer = tf.train.AdamOptimizer(learning_rate=1e-4) +grad = tfe.implicit_gradients(loss) +for i in range(20001): + with tf.device(device): + (inputs, targets) = data.train.next_batch(50) + optimizer.apply_gradients(grad(model, inputs, targets)) + if i % 100 == 0: + print("Step %d: Loss on training set : %f" % + (i, loss(model, inputs, targets).numpy())) +print("Loss on test set: %f" % loss(model, data.test.images, data.test.labels).numpy()) +``` + +For a more complete example, see +[`tensorflow/contrib/eager/python/examples/mnist.py`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist/mnist.py) + +### Checkpointing trained variables + +TensorFlow Variables (`tfe.Variable`) provides a way to represent shared, +persistent state of your model. The `tfe.Saver` class (which is a thin wrapper +over the +[`tf.train.Saver`](https://www.tensorflow.org/api_docs/python/tf/train/Saver) +class) provides a means to save and restore variables to and from _checkpoints_. + +For example: + +```python +# Create variables. +x = tfe.Variable(10., name='x') +y = tfe.Variable(5., name='y') + +# Create a Saver. +saver = tfe.Saver([x, y]) + +# Assign new values to the variables and save. +x.assign(2.) +saver.save('/tmp/ckpt') + +# Change the variable after saving. +x.assign(11.) +assert 16. == (x + y).numpy() # 11 + 5 + +# Restore the values in the checkpoint. +saver.restore('/tmp/ckpt') + +assert 7. == (x + y).numpy() # 2 + 5 +``` + +### `tfe.Network` + +You may often want to organize your models using classes, like the `MNISTModel` +class described above. We recommend inheriting from the `tfe.Network` class as +it provides conveniences like keeping track of all model variables and methods +to save and restore from checkpoints. + +Sub-classes of `tfe.Network` may register `Layer`s (like classes in +[`tf.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/layers), +or [Keras +layers](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/layers)) +using a call to `self.track_layer()` and define the computation in an +implementation of `call()`. + +Note that `tf.layers.Layer` objects (like `tf.layers.Dense`) create variables +lazily, when the first input is encountered. + +For example, consider the following two-layer neural network: + +```python +class TwoLayerNet(tfe.Network): + def __init__(self): + super(TwoLayerNet, self).__init__() + self.layer1 = self.track_layer( + tf.layers.Dense(2, activation=tf.nn.relu, use_bias=False)) + self.layer2 = self.track_layer(tf.layers.Dense(3, use_bias=False)) + + def call(self, x): + return self.layer2(self.layer1(x)) + +net = TwoLayerNet() + +# No variables created yet +assert 0 == len(net.variables) + +# They are created on first input: +inp = tf.constant([[1.]]) + +# Since input is a 1x1 matrix, net.l1 has 2 units and net.l2 has 3 units, +# the output is the product of a 1x1 matrix with a 1x2 matrix with a 2x3 +# matrix. +assert [1, 3] == net(inp).shape.as_list() # Invoke net; get output shape. +assert 1 == len(net.layer1.variables) +assert 1 == len(net.layer2.variables) +assert 2 == len(net.variables) # weights for each layer. +assert [1, 2] == net.variables[0].shape.as_list() # weights of layer1. +assert [2, 3] == net.variables[1].shape.as_list() # weights of layer2. +``` + +The `tfe.Network` class is itself a sub-class of `tf.layers.Layer`. This allows +instances of `tfe.Network` to be embedded in other networks. For example: + +```python +class ThreeLayerNet(tfe.Network): + def __init__(self): + super(ThreeLayerNet, self).__init__() + self.a = self.track_layer(TwoLayerNet()) + self.b = self.track_layer(tf.layers.Dense(4, use_bias=False)) + + def call(self, x): + return self.b(self.a(x)) + +net = ThreeLayerNet() + +assert [1, 4] == net(inp).shape.as_list() +assert 3 == len(net.variables) +assert [1, 2] == net.variables[0].shape.as_list() +assert [2, 3] == net.variables[1].shape.as_list() +assert [3, 4] == net.variables[2].shape.as_list() +``` + +See more examples in +[`tensorflow/contrib/eager/python/examples`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples). + +`tfe.Saver` in combination with `tfe.restore_variables_on_create` provides a +convenient way to save and load checkpoints without changing the program once +the checkpoint has been created. For example, we can set an objective for the +output of our network, choose an optimizer, and a location for the checkpoint: + +```python +objective = tf.constant([[2., 3., 4., 5.]]) +optimizer = tf.train.AdamOptimizer(0.01) +checkpoint_directory = '/tmp/tfe_example' +checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') +net = ThreeLayerNet() +``` + +Note that variables have not been created yet. We want them to be restored from +a checkpoint, if one exists, so we create them inside a +`tfe.restore_variables_on_create` context manager. Then our training loop is the +same whether starting training or resuming from a previous checkpoint: + +```python +with tfe.restore_variables_on_create( + tf.train.latest_checkpoint(checkpoint_directory)): + global_step = tf.train.get_or_create_global_step() + for _ in range(100): + loss_fn = lambda: tf.norm(net(inp) - objective) + optimizer.minimize(loss_fn, global_step=global_step) + if tf.equal(global_step % 20, 0): + print("Step %d, output %s" % (global_step.numpy(), + net(inp).numpy())) + all_variables = ( + net.variables + + tfe.get_optimizer_variables(optimizer) + + [global_step]) + # Save the checkpoint. + tfe.Saver(all_variables).save(checkpoint_prefix, global_step=global_step) +``` + +The first time it runs, `Network` variables are initialized randomly. Then the +output is trained to match the objective we've set: + +``` +Step 20, output [[ 0.03575622 0.29863232 0.03474367 0.24735749]] +Step 40, output [[ 0.40646029 0.9856872 0.46851286 0.95358551]] +Step 60, output [[ 1.74541104 2.800704 1.79055595 2.74783421]] +Step 80, output [[ 2.14977384 3.44340849 3.96120024 5.16242075]] +Step 100, output [[ 1.99943113 3.02364397 3.93500996 4.9610076 ]] +``` + +In subsequent iterations, variables are initialized with the values read from +the latest checkpoint. Running the same code again, we continue from where we +left off: + +``` +Step 120, output [[ 1.99234128 3.0271616 3.98732996 4.96401167]] +Step 140, output [[ 2.00133467 3.01270437 4.00616646 5.00406504]] +Step 160, output [[ 1.99647415 2.9956708 3.99064088 4.99632359]] +Step 180, output [[ 2.00699997 3.00904822 4.00706148 5.01193142]] +Step 200, output [[ 1.98334622 2.98249531 3.97375059 4.97123432]] +``` + + +### Summaries, metrics and TensorBoard + +[TensorBoard](https://www.tensorflow.org/get_started/summaries_and_tensorboard) +is a popular tool for understanding, debugging and optimizing the model training +process. To benefit from the visualizations offered by TensorBoard, summary +events need to be written during the course of execution of your program. You +might find many Tensorflow programs that include the +[`tf.summary`](https://www.tensorflow.org/api_guides/python/summary) operations +during graph construction. + +`tf.summary` operations are *not* compatible with eager execution, but an +equivalent alternative exists in +[`tf.contrib.summary`](https://www.tensorflow.org/versions/master/api_guides/python/tf/contrib/summary/) +that is compatible with both eager execution and graph construction. + +During model construction simply insert summary operations like +`tf.contrib.summary.scalar`. These operations do nothing by default, unless a +summary writer is currently active and a writing policy is set. + +For example, to record summaries once every 100 global steps, use: + +```python +tf.train.get_or_create_global_step() # Ensuring the global step variable exists +writer = tf.contrib.summary.create_summary_file_writer(logdir) + +for _ in range(iterations): + with writer.as_default(): + with tf.contrib.summary.record_summaries_every_n_global_steps(100): + # your model code goes here + tf.contrib.summary.scalar('loss', loss) + # ... +``` + +See the full mnist example in +[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist) +for a full model using `tf.contrib.summary`. + +Similarly to summaries, the metrics in `tf.metrics` are currently not compatible +with eager execution. We instead provide object-oriented metrics in the +`tfe.metrics` package, which are compatible with graph construction as well. + +Metrics in the `tfe.metrics`, such as `tfe.metrics.Mean` and +`tfe.Metrics.Accuracy`, all implement an intuitive object-oriented +interface. Here's an example of how to use the `tfe.metrics.Mean` metric: + +```python +# Metrics are objects, which can be created and destroyed. +my_mean = tfe.metrics.Mean(name='my_mean') +# While a metric is active, you can call it as a function to accumulate into its +# internal state. +my_mean(0.0) +my_mean(10.0) +# Once you've finished updating the metric, you can get its result. In this case +# a simple average over all the calls to it. If a summary writer is active the +# metric will write the appropriate summaries using the metric name. +assert 5.0 == my_mean.result().numpy() +``` + +For a full example of a model using metrics for evaluation, see the mnist +example in +[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist). + +### Input Pipelines + +The discussion above has been centered around the computation executed by your +model. The +[`tf.data`](https://www.tensorflow.org/versions/master/api_docs/python/tf/data) +module provides APIs to build complex input pipelines from simple, reusable +pieces. + +If you're familiar with constructing `tf.data.Dataset` objects when building +TensorFlow graphs, the same API calls are used when eager execution is enabled. +However, the process of iterating over elements of the dataset differs between +eager execution and graph construction. When eager execution is enabled, the +discussion on iterator creation using `make_one_shot_iterator()` and +`get_next()` in the +[Programmer's +Guide](https://www.tensorflow.org/versions/master/programmers_guide/datasets) is +*not* applicable. Instead, a more Pythonic `Iterator` class is available. + +For example: + +```python +# Create a source Dataset from in-memory numpy arrays. +# For reading from files on disk, you may want to use other Dataset classes +# like the TextLineDataset or the TFRecordDataset. +dataset = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]) + +# Apply transformations, shuffling, batching etc. +dataset = dataset.map(tf.square).shuffle(2).batch(2) + +# Use tfe.Iterator to iterate over the dataset. +for x in tfe.Iterator(dataset): + print(x) +``` + +Output: + +``` +tf.Tensor([4 9], shape=(2,), dtype=int32) +tf.Tensor([16 25], shape=(2,), dtype=int32) +tf.Tensor([36 1], shape=(2,), dtype=int32) +``` + +## Interoperating with Graphs + +Eager execution improves the process of model development in Python; however, +because it is in its earliest stages, it does not yet support some features +available to [TensorFlow +graphs](https://www.tensorflow.org/get_started/get_started#the_computational_graph) +that are desirable when deploying models in production. In particular, eager +execution does not yet support distributed training, exporting models (to other +[programming languages](https://www.tensorflow.org/api_docs/), [TensorFlow +serving](https://www.tensorflow.org/serving/), and mobile applications), and +various memory and computation optimizations that are applied to TensorFlow's +dataflow graphs. + +That said, the APIs used to build modes are exactly the same whether executing +eagerly or constructing graphs. This means that you can iteratively develop your +model with eager execution enabled and later, if needed, use the same code to +reap the benefits of representing models as computational graphs. + +For example, +[`mnist.py`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist/mnist.py) +defines a model that is eagerly executed. That same code is used to construct +and execute a graph in +[`mnist_graph_test.py`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py). + +Other models in the [examples +directory](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/) +demonstrate this as well. + +Some differences worth noting: + +- There is no notion of a `tf.placeholder` or a `tf.Session` when eager + execution is enabled. +- Many properties on the `tf.Tensor` object, like `tf.Tensor.name`, + `tf.Tensor.op`, `tf.Tensor.inputs` are not meaningful when eager execution + is enabled and their use will raise an `AttributeError`. +- To use `tfe.implicit_gradients` in graph construction, variables must be + created with [`use_resource=True`] provided to + [`tf.get_variable()`](https://www.tensorflow.org/api_docs/python/tf/get_variable) + or + [`tf.variable_scope()`](https://www.tensorflow.org/api_docs/python/tf/variable_scope). +- Some API calls (such as the functional-style `tf.layers.dense`, + `tf.layers.conv2d`) are not compatible with eager execution. Use of such + methods should raise an error indicating the alternative (e.g., the + `tf.layers.Dense` and `tf.layers.Conv2D` classes). + +## What next? + +Please give eager execution a spin. This feature is in early stages and is +evolving, so we welcome your feedback via issues on GitHub (see [known +issues](https://github.com/tensorflow/tensorflow/labels/eager)). + +You may want to browse through some sample code, including benchmarks for some: + +- [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression) +- [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist) +- [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50) +- [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot) +- [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb) + diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index cba8e89209..c6e577223f 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -153,6 +153,7 @@ sh_binary( "//tensorflow:tensorflow_py", "//tensorflow/contrib/boosted_trees:boosted_trees_pip", "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip", + "//tensorflow/contrib/eager/python/examples:examples_pip", "//tensorflow/contrib/gan:gan", "//tensorflow/contrib/graph_editor:graph_editor_pip", "//tensorflow/contrib/keras:keras", -- GitLab From c44f67a7ed5870fe8a1c0d6257ce597ca2ef7564 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Mon, 30 Oct 2017 23:23:18 -0700 Subject: [PATCH 558/573] Disable clang_format check. (#14115) Different clang_format version can cause different formats with the same style option. This check might be too strict. Disable for now. --- tensorflow/tools/ci_build/ci_sanity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index d42563553c..f1c207f9b6 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -505,8 +505,8 @@ do_check_load_py_test() { } # Supply all sanity step commands and descriptions -SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_clang_format_check") -SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Clang Format Check: Check .h and .cc files with Google C++ style") +SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check") +SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links") INCREMENTAL_FLAG="" DEFAULT_BAZEL_CONFIGS="--config=hdfs --config=gcp" -- GitLab From 9ee0cecec4707217c4fa3ebe8359d7e43d24da23 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 30 Oct 2017 23:52:52 -0700 Subject: [PATCH 559/573] Shrink the model size for unit test. PiperOrigin-RevId: 174001263 --- .../contrib/eager/python/examples/rnn_ptb/rnn_ptb.py | 11 +++++++++++ .../python/examples/rnn_ptb/rnn_ptb_graph_test.py | 2 +- .../eager/python/examples/rnn_ptb/rnn_ptb_test.py | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py index c67d77b386..30bb3c8ad3 100644 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py @@ -273,6 +273,17 @@ def large_model(use_cudnn_rnn): use_cudnn_rnn=use_cudnn_rnn) +def test_model(use_cudnn_rnn): + """Returns a tiny PTBModel for unit tests.""" + return PTBModel( + vocab_size=100, + embedding_dim=20, + hidden_dim=20, + num_layers=2, + dropout_ratio=0., + use_cudnn_rnn=use_cudnn_rnn) + + def main(_): tfe.enable_eager_execution() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py index 168b5c5356..63b5c4c54d 100644 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py @@ -41,7 +41,7 @@ class PTBTest(tf.test.TestCase): inputs = np.ones(inputs_ph.shape.as_list(), dtype=np.int64) labels = np.ones(labels_ph.shape.as_list(), dtype=np.int64) - model = rnn_ptb.small_model(tf.test.is_gpu_available()) + model = rnn_ptb.test_model(tf.test.is_gpu_available()) optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) loss = rnn_ptb.loss_fn(model, inputs_ph, labels_ph, training=True) grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py index 6f296c2aba..b279bc4a7c 100644 --- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py @@ -35,7 +35,7 @@ def device(): class PTBTest(tf.test.TestCase): def testTrain(self): - model = rnn_ptb.small_model(tfe.num_gpus() > 0) + model = rnn_ptb.test_model(tfe.num_gpus() > 0) sequence_length = 35 data = np.ones([4 * sequence_length, 20], dtype=np.int64) with tf.device(device()): @@ -45,7 +45,7 @@ class PTBTest(tf.test.TestCase): rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) def testApply(self): - model = rnn_ptb.small_model(tfe.num_gpus() > 0) + model = rnn_ptb.test_model(tfe.num_gpus() > 0) with tf.device(device()): model(tf.ones([35, 20], dtype=tf.int64), training=False) -- GitLab From 333ba224daee839ce569565b149e9d7a63b5c7e4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 31 Oct 2017 05:13:40 -0700 Subject: [PATCH 560/573] Dependency information for Skylark macros PiperOrigin-RevId: 174023371 --- tensorflow/tensorflow.bzl | 61 ++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 3001a37473..e647a78055 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -1,13 +1,6 @@ # -*- Python -*- -# Given a source file, generate a test name. -# i.e. "common_runtime/direct_session_test.cc" becomes -# "common_runtime_direct_session_test" -def src_to_test_name(src): - return src.replace("/", "_").split(".")[0] - - # Return the options to use for a C++ library or binary build. # Uses the ":optmode" config_setting to pick the options. load( @@ -16,16 +9,30 @@ load( "tf_sycl_tests_tags", "tf_additional_xla_deps_py", "if_static",) -load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda", "cuda_default_copts") +load( + "@local_config_cuda//cuda:build_defs.bzl", + "if_cuda", + "cuda_default_copts",) load( "//third_party/mkl:build_defs.bzl", "if_mkl",) +def register_extension_info(**kwargs): + pass + + +# Given a source file, generate a test name. +# i.e. "common_runtime/direct_session_test.cc" becomes +# "common_runtime_direct_session_test" +def src_to_test_name(src): + return src.replace("/", "_").split(".")[0] + def full_path(relative_paths): return [PACKAGE_NAME + "/" + relative for relative in relative_paths] + # List of proto files for android builds def tf_android_core_proto_sources(core_proto_sources_relative): return [ @@ -290,6 +297,11 @@ def tf_cc_binary(name, linkopts=linkopts + _rpath_linkopts(name), **kwargs) +register_extension_info( + extension_name="tf_cc_binary", + label_regex_for_dep="{extension_name}.*") + + def tf_gen_op_wrapper_cc(name, out_ops_file, pkg="", @@ -551,6 +563,10 @@ def tf_cc_test(name, nocopts=nocopts, **kwargs) +register_extension_info( + extension_name="tf_cc_test", + label_regex_for_dep="{extension_name}.*") + # Part of the testing workflow requires a distinguishable name for the build # rules that involve a GPU, even if otherwise identical to the base rule. @@ -793,6 +809,11 @@ def tf_cuda_library(deps=None, cuda_deps=None, copts=None, **kwargs): copts=copts + if_cuda(["-DGOOGLE_CUDA=1"]) + if_mkl(["-DINTEL_MKL=1"]), **kwargs) +register_extension_info( + extension_name="tf_cuda_library", + label_regex_for_dep="{extension_name}") + + def tf_kernel_library(name, prefix=None, @@ -862,6 +883,10 @@ def tf_kernel_library(name, deps=deps, **kwargs) +register_extension_info( + extension_name="tf_kernel_library", + label_regex_for_dep="{extension_name}(_gpu)?") + def tf_mkl_kernel_library(name, prefix=None, @@ -1165,6 +1190,10 @@ def tf_custom_op_py_library(name, visibility=visibility, deps=deps,) +register_extension_info( + extension_name="tf_custom_op_py_library", + label_regex_for_dep="{extension_name}") + def tf_extension_linkopts(): return [] # No extension link opts @@ -1250,6 +1279,10 @@ def py_test(deps=[], **kwargs): }), **kwargs) +register_extension_info( + extension_name="py_test", + label_regex_for_dep="{extension_name}") + def tf_py_test(name, srcs, @@ -1284,6 +1317,10 @@ def tf_py_test(name, flaky=flaky, srcs_version="PY2AND3") +register_extension_info( + extension_name="tf_py_test", + label_regex_map={"deps": "additional_deps:{extension_name}"}) + def cuda_py_test(name, srcs, @@ -1310,6 +1347,10 @@ def cuda_py_test(name, flaky=flaky, xla_enabled=xla_enabled) +register_extension_info( + extension_name="cuda_py_test", + label_regex_map={"additional_deps": "additional_deps:{extension_name}"}) + def sycl_py_test(name, srcs, @@ -1336,6 +1377,10 @@ def sycl_py_test(name, flaky=flaky, xla_enabled=xla_enabled) +register_extension_info( + extension_name="sycl_py_test", + label_regex_map={"additional_deps": "additional_deps:{extension_name}"}) + def py_tests(name, srcs, -- GitLab From 648993e8239655a47dddee5ead864b204c9e1042 Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Tue, 31 Oct 2017 10:52:59 -0400 Subject: [PATCH 561/573] delete extraneous file --- tensorflow/contrib/eager/README.OPENSOURCE.md | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 tensorflow/contrib/eager/README.OPENSOURCE.md diff --git a/tensorflow/contrib/eager/README.OPENSOURCE.md b/tensorflow/contrib/eager/README.OPENSOURCE.md deleted file mode 100644 index a4a3af08cf..0000000000 --- a/tensorflow/contrib/eager/README.OPENSOURCE.md +++ /dev/null @@ -1,15 +0,0 @@ -TensorFlow has many kernels for doing (deep) learning and data manipulation. -There are typically assembled into computational graphs which can run -efficiently in a variety of environments. - -We are exploring an alternative interaction, where kernels are invoked -immediately and call this "eager execution". We are hoping to retain the -benefits of graphs while improving usability with benefits like: - -- Immediate error messages and easier debugging -- Flexibility to use Python datastructures and control flow -- Reduced boilerplate - -Eager execution is under active development. -There are not many developer-facing materials yet, but stay tuned for updates -in this directory. -- GitLab From 123749fb1823a79c02446c775c46ac22afd020d4 Mon Sep 17 00:00:00 2001 From: "Yuan (Terry) Tang" Date: Tue, 31 Oct 2017 12:28:08 -0400 Subject: [PATCH 562/573] Remove Scikit Flow link and description (#14036) --- tensorflow/docs_src/community/welcome.md | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md index c4f78051f0..33740de5d5 100644 --- a/tensorflow/docs_src/community/welcome.md +++ b/tensorflow/docs_src/community/welcome.md @@ -20,7 +20,6 @@ The TensorFlow community has created many great projects around TensorFlow, incl * [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com) * [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow) * [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials) -* [Scikit Flow - Simplified Interface for TensorFlow](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/learn/python/learn) * [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow) * [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0)) * [Rust language bindings](https://github.com/google/tensorflow-rust) -- GitLab From c37ebf0d5364b4393ba023f6ab1f9d75216182f9 Mon Sep 17 00:00:00 2001 From: Thomas Deegan Date: Tue, 31 Oct 2017 10:13:16 -0700 Subject: [PATCH 563/573] Resolve //tensorflow relative to tensorflow repo so that tfcompile.bzl can be correctly loaded from another Bazel project (#14103) --- tensorflow/compiler/aot/tfcompile.bzl | 74 +++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl index 2adb1dc65e..363d6925a1 100644 --- a/tensorflow/compiler/aot/tfcompile.bzl +++ b/tensorflow/compiler/aot/tfcompile.bzl @@ -4,7 +4,7 @@ To use from your BUILD file, add the following line to load the macro: -load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") +load("@org_tensorflow//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") Then call the macro like this: @@ -16,14 +16,14 @@ tf_library( ) """ -load("//tensorflow:tensorflow.bzl", "if_android", "tf_copts") +load("@org_tensorflow//tensorflow:tensorflow.bzl", "if_android", "tf_copts") def tf_library(name, graph, config, freeze_checkpoint=None, freeze_saver=None, cpp_class=None, gen_test=True, gen_benchmark=True, visibility=None, testonly=None, tfcompile_flags=None, - tfcompile_tool="//tensorflow/compiler/aot:tfcompile", + tfcompile_tool="@org_tensorflow//tensorflow/compiler/aot:tfcompile", include_standard_runtime_deps=True, deps=None, tags=None): """Runs tfcompile to compile a TensorFlow graph into executable code. @@ -119,9 +119,9 @@ def tf_library(name, graph, config, out_nodes_file, ] + freeze_saver_srcs, outs=[freeze_file], - cmd=("$(location //tensorflow/python/tools:freeze_graph)" + + cmd=("$(location @org_tensorflow//tensorflow/python/tools:freeze_graph)" + freeze_args), - tools=["//tensorflow/python/tools:freeze_graph"], + tools=["@org_tensorflow//tensorflow/python/tools:freeze_graph"], tags=tags, ) tfcompile_graph = freeze_file @@ -207,22 +207,22 @@ def tf_library(name, graph, config, # These deps are required by all tf_library targets even if # include_standard_runtime_deps is False. Without them, the # generated code will fail to compile. - "//tensorflow/compiler/tf2xla:xla_compiled_cpu_function", - "//tensorflow/core:framework_lite", + "@org_tensorflow//tensorflow/compiler/tf2xla:xla_compiled_cpu_function", + "@org_tensorflow//tensorflow/core:framework_lite", ] + (need_xla_data_proto and [ # If we're generating the program shape, we must depend on the proto. - "//tensorflow/compiler/xla:xla_data_proto", + "@org_tensorflow//tensorflow/compiler/xla:xla_data_proto", ] or []) + (include_standard_runtime_deps and [ # TODO(cwhipkey): only depend on kernel code that the model actually needed. - "//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_1d", - "//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_2d", - "//tensorflow/compiler/xla/service/cpu:cpu_runtime_avx", - "//tensorflow/compiler/xla/service/cpu:cpu_runtime_neon", - "//tensorflow/compiler/xla/service/cpu:cpu_runtime_sse4_1", - "//tensorflow/compiler/xla/service/cpu:runtime_conv2d", - "//tensorflow/compiler/xla/service/cpu:runtime_matmul", - "//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_conv2d", - "//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_matmul", + "@org_tensorflow//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_1d", + "@org_tensorflow//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_2d", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:cpu_runtime_avx", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:cpu_runtime_neon", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:cpu_runtime_sse4_1", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_conv2d", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_matmul", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_conv2d", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_matmul", "//third_party/eigen3", ] or []) + (deps or []), tags=tags, @@ -248,12 +248,12 @@ def tf_library(name, graph, config, name=("gen_" + test_name), testonly=1, srcs=[ - "//tensorflow/compiler/aot:test.cc", + "@org_tensorflow//tensorflow/compiler/aot:test.cc", header_file, ], outs=[test_file], cmd=("sed " + sed_replace + - " $(location //tensorflow/compiler/aot:test.cc) " + + " $(location @org_tensorflow//tensorflow/compiler/aot:test.cc) " + "> $(OUTS)"), tags=tags, ) @@ -264,13 +264,13 @@ def tf_library(name, graph, config, srcs=[test_file], deps=[ ":" + name, - "//tensorflow/compiler/tf2xla:xla_local_runtime_context", - "//tensorflow/compiler/aot:runtime", - "//tensorflow/compiler/aot:tf_library_test_main", - "//tensorflow/compiler/xla:executable_run_options", + "@org_tensorflow//tensorflow/compiler/tf2xla:xla_local_runtime_context", + "@org_tensorflow//tensorflow/compiler/aot:runtime", + "@org_tensorflow//tensorflow/compiler/aot:tf_library_test_main", + "@org_tensorflow//tensorflow/compiler/xla:executable_run_options", "//third_party/eigen3", - "//tensorflow/core:lib", - "//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", ], tags=tags, ) @@ -278,7 +278,7 @@ def tf_library(name, graph, config, if gen_benchmark: benchmark_name = name + "_benchmark" benchmark_file = benchmark_name + ".cc" - benchmark_main = ("//tensorflow/compiler/aot:" + + benchmark_main = ("@org_tensorflow//tensorflow/compiler/aot:" + "benchmark_main.template") # Rule to rewrite benchmark.cc to produce the benchmark_file. @@ -310,13 +310,13 @@ def tf_library(name, graph, config, linkopts = if_android(["-pie", "-s"]), deps=[ ":" + name, - "//tensorflow/compiler/tf2xla:xla_local_runtime_context", - "//tensorflow/compiler/aot:benchmark", - "//tensorflow/compiler/aot:runtime", - "//tensorflow/compiler/xla:executable_run_options", + "@org_tensorflow//tensorflow/compiler/tf2xla:xla_local_runtime_context", + "@org_tensorflow//tensorflow/compiler/aot:benchmark", + "@org_tensorflow//tensorflow/compiler/aot:runtime", + "@org_tensorflow//tensorflow/compiler/xla:executable_run_options", "//third_party/eigen3", ] + if_android([ - "//tensorflow/compiler/aot:benchmark_extra_android", + "@org_tensorflow//tensorflow/compiler/aot:benchmark_extra_android", ]), tags=tags, ) @@ -326,11 +326,11 @@ def target_llvm_triple(): # TODO(toddw): Add target_triple for other targets. For details see: # http://llvm.org/docs/doxygen/html/Triple_8h_source.html return select({ - "//tensorflow:android_armeabi": "armv5-none-android", - "//tensorflow:android_arm": "armv7-none-android", - "//tensorflow:android_arm64": "aarch64-none-android", - "//tensorflow:android_x86": "i686-none-android", - "//tensorflow:linux_ppc64le": "ppc64le-ibm-linux-gnu", - "//tensorflow:darwin": "x86_64-none-darwin", + "@org_tensorflow//tensorflow:android_armeabi": "armv5-none-android", + "@org_tensorflow//tensorflow:android_arm": "armv7-none-android", + "@org_tensorflow//tensorflow:android_arm64": "aarch64-none-android", + "@org_tensorflow//tensorflow:android_x86": "i686-none-android", + "@org_tensorflow//tensorflow:linux_ppc64le": "ppc64le-ibm-linux-gnu", + "@org_tensorflow//tensorflow:darwin": "x86_64-none-darwin", "//conditions:default": "x86_64-pc-linux", }) -- GitLab From ad7bb2b9ebe60afc523a9571a0735d8a39df1cb6 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 31 Oct 2017 10:17:51 -0700 Subject: [PATCH 564/573] eager: Update broken links in guide.md (#14135) --- tensorflow/contrib/eager/python/g3doc/guide.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md index e945bc20f4..e76745a807 100644 --- a/tensorflow/contrib/eager/python/g3doc/guide.md +++ b/tensorflow/contrib/eager/python/g3doc/guide.md @@ -68,9 +68,9 @@ enabled. A significant fraction of the [TensorFlow API](https://www.tensorflow.org/api_docs/python/) consists of numerical operations: -[arithmetic operations](https://www.tensorflow.org/api_docs/python/tf/matmul), -[matrix operations](https://www.tensorflow.org/api_docs/python/tf/matmul), -[linear algebra operations](https://www.tensorflow.org/api_docs/python/tf/linalg), +[arithmetic operations](https://www.tensorflow.org/api_guides/python/math_ops#Arithmetic_Operators), +[matrix operations](https://www.tensorflow.org/api_guides/python/math_ops#Matrix_Math_Functions), +[linear algebra operations](https://www.tensorflow.org/versions/master/api_docs/python/tf/linalg), etc. With eager execution enabled, these operations consume and return @@ -746,7 +746,7 @@ during graph construction. `tf.summary` operations are *not* compatible with eager execution, but an equivalent alternative exists in -[`tf.contrib.summary`](https://www.tensorflow.org/versions/master/api_guides/python/tf/contrib/summary/) +[`tf.contrib.summary`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/summary) that is compatible with both eager execution and graph construction. During model construction simply insert summary operations like @@ -887,7 +887,7 @@ Some differences worth noting: Please give eager execution a spin. This feature is in early stages and is evolving, so we welcome your feedback via issues on GitHub (see [known -issues](https://github.com/tensorflow/tensorflow/labels/eager)). +issues](https://github.com/tensorflow/tensorflow/labels/comp:eager)). You may want to browse through some sample code, including benchmarks for some: -- GitLab From 479ee24a0a5bb842d61f86092ab609f4918525f4 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 31 Oct 2017 10:18:15 -0700 Subject: [PATCH 565/573] eager: Update broken link in README (#14136) --- tensorflow/contrib/eager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md index db11dbb0d7..ae4b07799f 100644 --- a/tensorflow/contrib/eager/README.md +++ b/tensorflow/contrib/eager/README.md @@ -35,7 +35,7 @@ print(m) This feature is in early stages and work remains to be done in terms of smooth support for distributed and multi-GPU training and CPU performance. -- [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Aproj%3Aeager) +- [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Acomp%3Aeager) - Feedback is welcome, please consider [filing an issue](https://github.com/tensorflow/tensorflow/issues/new) to provide it. -- GitLab From 9a2b0983a94ab886fcf968796c4ecdf32595a590 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Tue, 31 Oct 2017 10:18:44 -0700 Subject: [PATCH 566/573] Add apt-key for ubuntu keyserver (#14114) --- tensorflow/tools/ci_build/install/install_deb_packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh index 03ca4716b4..4ab307c925 100755 --- a/tensorflow/tools/ci_build/install/install_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh @@ -28,6 +28,7 @@ if [[ "$1" != "" ]] && [[ "$1" != "--without_cmake" ]]; then fi # Install dependencies from ubuntu deb repository. +apt-key adv --keyserver keyserver.ubuntu.com --recv 084ECFC5828AB726 apt-get update if [[ "$ubuntu_version" == "14" ]]; then -- GitLab From 6eac524ef63728bdc10c40f95d30c94aede5f4ea Mon Sep 17 00:00:00 2001 From: cglewis Date: Tue, 31 Oct 2017 10:56:48 -0700 Subject: [PATCH 567/573] Use 'LABEL maintainer=' in Dockerfile * Use 'LABEL maintainer=' in Dockerfile This fix is a follow up of 13961 to replace `MAINTAINER` with `LABEL maintainer=` in Dockerfile. The keyword `MAINTAINER` has long been deprecated and is replaced by `LABEL`, which is much more flexible and is easily searchable through `docker inspect`. This fix replaces remaining `MAINTAINER` with `LABEL`. Signed-off-by: Charlie Lewis * Additional `MAITAINER` -> `LABEL` Signed-off-by: Charlie Lewis --- tensorflow/contrib/makefile/Dockerfile | 2 +- tensorflow/examples/udacity/Dockerfile | 2 +- tensorflow/tools/ci_build/Dockerfile.pi-python3 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/makefile/Dockerfile b/tensorflow/contrib/makefile/Dockerfile index 341f22e692..64d571a4ed 100644 --- a/tensorflow/contrib/makefile/Dockerfile +++ b/tensorflow/contrib/makefile/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:16.04 -MAINTAINER Gunhan Gulsoy +LABEL maintainer="Gunhan Gulsoy " # Install make build dependencies for TensorFlow. RUN apt-get update diff --git a/tensorflow/examples/udacity/Dockerfile b/tensorflow/examples/udacity/Dockerfile index 3d48ced41b..3ca58566c1 100644 --- a/tensorflow/examples/udacity/Dockerfile +++ b/tensorflow/examples/udacity/Dockerfile @@ -1,5 +1,5 @@ FROM gcr.io/tensorflow/tensorflow:latest -MAINTAINER Vincent Vanhoucke +LABEL maintainer="Vincent Vanhoucke " # Pillow needs libjpeg by default as of 3.0. RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3 index 18b131ea19..b1c648ba30 100644 --- a/tensorflow/tools/ci_build/Dockerfile.pi-python3 +++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3 @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ -- GitLab From e6faa845c51bb69465146d93646947fd2ba53efa Mon Sep 17 00:00:00 2001 From: Michael Case Date: Tue, 31 Oct 2017 11:05:14 -0700 Subject: [PATCH 568/573] Merge v1.4-rc1 back into master branch. (#13960) * Update RELEASE NOTES for TensorFlow 1.4 * Update the version strings for TF 1.4-rc0. * Update version strings in POM files missed by update script. * Pin TensorBoard 0.4 to TensorFlow 1.4 * Fixing the name of the disabled test. (#13592) * Revert "Implementing ghost batch norm as defined in https://arxiv.org/pdf/1705.08741." This reverts commit 125f7afa4a483855dc75791445d2dea64587876a. * Disable iterator_ops_test on Windows for 1.4 release (#13609) * Disable failing Windows tests for r1.4 release. testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU test is failing with "TypeError: only integer scalar arrays can be converted to a scalar index" on the Windows GPU Release bot. Disabling test. * Fix typo. * Also disalbe iterator_ops_test from contrib/. * Add contributing authors to 1.4 Release notes. Thanks! * Fixes to authors. Removed duplicate and removed googler from contributing author list. * Fixes and additions to release notes. Added line about Keras moving into core. Added line about CUDA/cuDNN versions. Added line about custom ops. * Fixing a master regression (#13562) * Update version strings for 1.4.0rc1 * Remaining cherry-picks for 1.4.0rc1 (#13700) * Java: Tweak to address some Javadoc errors. PiperOrigin-RevId: 171987329 * Fix S3 BUILD not including files explicitly. This causes remote builds to fail since they AWS headers were missing. PiperOrigin-RevId: 171718021 * Add missing default config setting in aws.BUILD (#13662) * Remove setting AWS logging for S3 file system. Was causing issues with tests. Can repro test failures on Macs by running... bazel test --config=s3 --cache_test_results=no --test_output=streamed //tensorflow/core/kernels:control_flow_ops_test Possible reason for error is symbol collision with AWS logging code. One possible solution would be to split out another shared object for the S3 filesystem op which does not link in libtensorflow_framework.so. This is done, for example, by libforestprotos.so in tensorflow/contrib/tensor_forest/BUILD PiperOrigin-RevId: 171246381 * Relanding change to add config to enable S3 file system support. Pass --config=s3 argument to Bazel to build with S3 file system support. Change was originally rolled back due to a failure it caused in //tensorflow/core/kernels:control_flow_ops_test on Macs which is now fixed. PiperOrigin-RevId: 171579378 * Update release notes about Amazon S3 file system support being default. * Add documentation to sloppy_interleave function PiperOrigin-RevId: 171303413 * Add `cudnn_rnn_ops` to the Windows build Fixes #13696. * Creating a patch for the wrong links that still point to dev. (#13753) * tfdbg release notes in r1.4 * Fix ambiguous type comparison in s3_crypto.cc (#13758) tensorflow/contrib/s3/s3_crypto.cc(74): error C2666: 'std::fpos<_Mbstatet>::operator ==': 3 overloads have similar conversions could be 'bool std::fpos<_Mbstatet>::operator ==(std::streamoff) const' or 'bool std::fpos<_Mbstatet>::operator ==(const std::fpos<_Mbstatet> &) We were seeing this compilation error on Windows builds. * Set estimator run_config default random seed to None. This will make it aligned with other parts of the TF. Many users are not aware of impact of non-random seed. For example it may lead to train only on a small fraction of training data due to preemptions. We're changing default behavior since we consider it as a bug fix. PiperOrigin-RevId: 172519268 * Move global_step_read dependency to model_fn instead of input_fn. PiperOrigin-RevId: 172366972 * [tf.data] Fix broken implementation of `Dataset.from_generator()` on Windows. Due to a mix-up between NumPy's default array element type for a Python `int` on Windows and Linux, a tf.py_func() in `Dataset.from_generator()` would appear to return the wrong type on Windows (np.int32 instead of np.int64). All code using `Dataset.from_generator()` on Windows was previously broken. This change fixes both `tf.data.Dataset.from_generator()` and `tf.contrib.data.Dataset.from_generator()`. It also enables test coverage for this method on Windows, which should prevent future breakage. PiperOrigin-RevId: 172346533 * Update RELEASE notes for change to run_config random seed. * Disable probable timeout flake on Ubuntu machines. PiperOrigin-RevId: 172408922 * Disabling failing contrib tests. * Disable S3 on Windows due to build issues. * Update serving_input_fn argument name to serving_input_receiver_fn PiperOrigin-RevId: 172787460 * Update the C++ API guide (#13858) - Adds the standard warning at the top that people may want the master branch - Includes a documentation fix for 1.4 (cc_binary -> tf_cc_binary to avoid undefined symbols). * Add known Dataset issue to RELEASE.md. (#13870) Adding info about issue using Unicode strings with Datasets. * Fixes to merge. * Fix spelling of tensorflow in install_sources.md --- RELEASE.md | 21 ++++++++++++++++-- configure.py | 1 + tensorflow/core/public/version.h | 2 +- tensorflow/docs_src/api_guides/cc/guide.md | 18 ++++++++++++--- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 18 +++++++-------- tensorflow/docs_src/install/install_linux.md | 22 +++++++++---------- tensorflow/docs_src/install/install_mac.md | 10 ++++----- .../docs_src/install/install_sources.md | 14 ++++++------ tensorflow/tools/ci_build/update_version.py | 2 +- tensorflow/tools/pip_package/setup.py | 2 +- 12 files changed, 72 insertions(+), 42 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 4a33bce8b2..d8db1f7200 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,6 +19,14 @@ (with GPU and gradient support). * Add a self-check on `import tensorflow` for Windows DLL issues. * Add NCHW support to `tf.depth_to_space` on GPU. +* TensorFlow Debugger (tfdbg): + * Add `eval` command to allow evaluation of arbitrary Python/numpy expressions + in tfdbg command-line interface. See + [Debugging TensorFlow Programs](https://www.tensorflow.org/programmers_guide/debugger) + for more details. + * Usability improvement: The frequently used tensor filter `has_inf_or_nan` is + now added to `Session` wrappers and hooks by default. So there is no need + for clients to call `.add_tensor_filter(tf_debug.has_inf_or_nan)` anymore. * SinhArcsinh (scalar) distribution added to `contrib.distributions`. * Make `GANEstimator` opensource. * `Estimator.export_savedmodel()` now includes all valid serving signatures @@ -60,10 +68,14 @@ * Fix `tf.contrib.distributions.Affine` incorrectly computing log-det-jacobian. * Fix `tf.random_gamma` incorrectly handling non-batch, scalar draws. * Resolved a race condition in TensorForest TreePredictionsV4Op. -* Google Cloud Storage file system and Hadoop file system support are now - default build options. +* Google Cloud Storage file system, Amazon S3 file system, and Hadoop file + system support are now default build options. * Custom op libraries must link against libtensorflow_framework.so (installed at `tf.sysconfig.get_lib()`). +* Change `RunConfig` default behavior to not set a random seed, making random + behavior independently random on distributed workers. We expect this to + generally improve training performance. Models that do rely on determinism + should set a random seed explicitly. ## Breaking Changes to the API * The signature of the `tf.contrib.data.rejection_resample()` function has been @@ -74,6 +86,11 @@ * Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`. * Reorder some TFGAN loss functions in a non-backwards compatible way. +## Known Issues +* In Python 3, `Dataset.from_generator()` does not support Unicode strings. + You must convert any strings to bytes objects before yielding them from + the generator. + ## Thanks to our Contributors This release contains contributions from many people at Google, as well as: diff --git a/configure.py b/configure.py index 425eae676c..bc7859fee4 100644 --- a/configure.py +++ b/configure.py @@ -994,6 +994,7 @@ def main(): environ_cp['TF_NEED_HDFS'] = '0' environ_cp['TF_NEED_JEMALLOC'] = '0' environ_cp['TF_NEED_OPENCL'] = '0' + environ_cp['TF_NEED_S3'] = '0' environ_cp['TF_CUDA_CLANG'] = '0' if is_macos(): diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 5d2298f7b7..0bdd0c52ca 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -24,7 +24,7 @@ limitations under the License. // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") -#define TF_VERSION_SUFFIX "-rc0" +#define TF_VERSION_SUFFIX "-rc1" #define TF_STR_HELPER(x) #x #define TF_STR(x) TF_STR_HELPER(x) diff --git a/tensorflow/docs_src/api_guides/cc/guide.md b/tensorflow/docs_src/api_guides/cc/guide.md index f30bf3797e..81fb1e1fda 100644 --- a/tensorflow/docs_src/api_guides/cc/guide.md +++ b/tensorflow/docs_src/api_guides/cc/guide.md @@ -1,4 +1,12 @@ # C++ API + +Note: By default [tensorflow.org](http://tensorflow.org) shows docs for the +most recent stable version. The instructions in this doc require building from +source. You will probably want to build from the `master` version of tensorflow. +You should, as a result, be sure you are following the +[`master` version of this doc](https://www.tensorflow.org/versions/master/api_guides/cc/guide), +in case there have been any changes. + [TOC] TensorFlow's C++ API provides mechanisms for constructing and executing a data @@ -48,7 +56,9 @@ TensorFlow `BUILD` file in the same directory with the following contents: ```python -cc_binary( +load("//tensorflow:tensorflow.bzl", "tf_cc_binary") + +tf_cc_binary( name = "example", srcs = ["example.cc"], deps = [ @@ -59,8 +69,10 @@ cc_binary( ) ``` -You should be able to build and run the example using the following command -(be sure to run `./configure` in your build sandbox first): +Use `tf_cc_binary` rather than Bazel's native `cc_binary` to link in necessary +symbols from `libtensorflow_framework.so`. You should be able to build and run +the example using the following command (be sure to run `./configure` in your +build sandbox first): ```shell bazel run -c opt //tensorflow/cc/example:example diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 70f756b194..3a153e8114 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc1.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index eca2ecc5ac..df43255896 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc1.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index 8eaec3712a..f7f2c3cdc7 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.4.0-rc0 + 1.4.0-rc1 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.4.0-rc0 + 1.4.0-rc1 @@ -124,7 +124,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc1.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -143,7 +143,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc1.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -151,10 +151,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc1.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc0.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc1.zip). 3. Extract this .zip file. @@ -202,7 +202,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.4.0-rc0.jar HelloTF.java
+
javac -cp libtensorflow-1.4.0-rc1.jar HelloTF.java
### Running @@ -216,11 +216,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.4.0-rc0.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.4.0-rc1.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.4.0-rc0.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.4.0-rc1.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 238159c6b1..414ab7b1f7 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -188,7 +188,7 @@ Take the following steps to install TensorFlow with Virtualenv: Virtualenv environment: 
(tensorflow)$ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl If you encounter installation problems, see [Common Installation Problems](#common_installation_problems). @@ -293,7 +293,7 @@ take the following steps: 
      $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl
If this step fails, see @@ -480,7 +480,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl @@ -648,14 +648,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -667,14 +667,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -686,14 +686,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp35-cp35m-linux_x86_64.whl
@@ -705,14 +705,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index a44ade0731..9a95710bfa 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -114,7 +114,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -235,7 +235,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -344,7 +344,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl @@ -517,7 +517,7 @@ This section documents the relevant values for Mac OS installations. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl
@@ -525,7 +525,7 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 1103c6a18e..6d0dcdcd4a 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -355,10 +355,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.4.0rc0 on Linux: +for TensorFlow 1.4.0rc1 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc0-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc1-py2-none-any.whl
## Validate your installation @@ -447,8 +447,8 @@ Stack Overflow and specify the `tensorflow` tag. **Linux** - - + + @@ -460,7 +460,7 @@ Stack Overflow and specify the `tensorflow` tag. **Mac**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.4.0rc0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.568
tensorflow-1.4.0rc1CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.4.0rc1GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.568
tensorflow-1.2.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.2.0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.55.18
tensorflow-1.1.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.2N/AN/A
- + @@ -471,8 +471,8 @@ Stack Overflow and specify the `tensorflow` tag. **Windows**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.4.0rc1CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.2.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.1.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
tensorflow_gpu-1.1.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
- - + + diff --git a/tensorflow/tools/ci_build/update_version.py b/tensorflow/tools/ci_build/update_version.py index c7841f35aa..d2a63e5d66 100755 --- a/tensorflow/tools/ci_build/update_version.py +++ b/tensorflow/tools/ci_build/update_version.py @@ -17,7 +17,7 @@ # Automatically update TensorFlow version in source files # # Usage: -# ./tensorflow/tools/ci_build/update_version.py --version 1.4.0-rc0 +# ./tensorflow/tools/ci_build/update_version.py --version 1.4.0-rc1 # ./tensorflow/tools/ci_build/update_version.py --nightly # """Update version of TensorFlow script.""" diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 4f0de8f768..071b3a2a18 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -29,7 +29,7 @@ from setuptools.dist import Distribution # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.4.0-rc0' +_VERSION = '1.4.0-rc1' REQUIRED_PACKAGES = [ 'enum34 >= 1.1.6', -- GitLab From f567ddf87d8d85e6cefa441283d20d52aac9f7fb Mon Sep 17 00:00:00 2001 From: Alex Sergeev Date: Tue, 31 Oct 2017 13:18:17 -0700 Subject: [PATCH 569/573] Add tf.sysconfig.get_compile_flags() & tf.sysconfig.get_link_flags() for custom operators (#13496) * Add flags for custom op compilation * Move ABI logic into version_info.cc * Add #include to be able to read _GLIBCXX_USE_CXX11_ABI value. * Make flags to be lists * Add _flag to cxx11_abi * Address review comment. * Move CXX import to the top level. * Add goldens update --- tensorflow/core/public/version.h | 2 ++ tensorflow/python/__init__.py | 2 ++ tensorflow/python/client/tf_session.i | 3 ++ tensorflow/python/framework/versions.py | 4 +++ tensorflow/python/platform/sysconfig.py | 29 +++++++++++++++++++ tensorflow/python/pywrap_tensorflow.py | 1 + tensorflow/tools/api/golden/tensorflow.pbtxt | 4 +++ .../api/golden/tensorflow.sysconfig.pbtxt | 8 +++++ tensorflow/tools/git/gen_git_source.py | 8 +++++ tensorflow/tools/git/gen_git_source.sh | 8 +++++ 10 files changed, 69 insertions(+) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 0bdd0c52ca..95ada559fd 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -116,5 +116,7 @@ extern const char* tf_compiler_version(); // The git commit designator when tensorflow was built // If no git repository, this will be "internal". extern const char* tf_git_version(); +// Value of the _GLIBCXX_USE_CXX11_ABI flag, or -1 if it's not set. +extern const int tf_cxx11_abi_flag(); #endif // TENSORFLOW_CORE_PUBLIC_VERSION_H_ diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py index 8d9c5de9ad..af34aca3e3 100644 --- a/tensorflow/python/__init__.py +++ b/tensorflow/python/__init__.py @@ -262,6 +262,7 @@ _allowed_symbols.extend([ 'VERSION', 'GIT_VERSION', 'COMPILER_VERSION', + 'CXX11_ABI_FLAG', ]) # Remove all extra symbols that don't have a docstring or are not explicitly @@ -280,6 +281,7 @@ _exported_dunders = set([ '__version__', '__git_version__', '__compiler_version__', + '__cxx11_abi_flag__', ]) # Expose symbols minus dunders, unless they are whitelisted above. diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index 4200439dc6..40731aba7d 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -45,6 +45,9 @@ tensorflow::ImportNumpy(); // Compiler %constant const char* __compiler_version__ = tf_compiler_version(); +// _GLIBCXX_USE_CXX11_ABI flag value +%constant const int __cxx11_abi_flag__ = tf_cxx11_abi_flag(); + // Release the Python GIL for the duration of most methods. %exception { Py_BEGIN_ALLOW_THREADS; diff --git a/tensorflow/python/framework/versions.py b/tensorflow/python/framework/versions.py index f4b01635dc..81529e2b1e 100644 --- a/tensorflow/python/framework/versions.py +++ b/tensorflow/python/framework/versions.py @@ -24,10 +24,12 @@ from tensorflow.python import pywrap_tensorflow __version__ = pywrap_tensorflow.__version__ __git_version__ = pywrap_tensorflow.__git_version__ __compiler_version__ = pywrap_tensorflow.__compiler_version__ +__cxx11_abi_flag__ = pywrap_tensorflow.__cxx11_abi_flag__ VERSION = __version__ GIT_VERSION = __git_version__ COMPILER_VERSION = __compiler_version__ +CXX11_ABI_FLAG = __cxx11_abi_flag__ GRAPH_DEF_VERSION = pywrap_tensorflow.GRAPH_DEF_VERSION GRAPH_DEF_VERSION_MIN_CONSUMER = ( @@ -39,7 +41,9 @@ __all__ = [ "__version__", "__git_version__", "__compiler_version__", + "__cxx11_abi_flag__", "COMPILER_VERSION", + "CXX11_ABI_FLAG", "GIT_VERSION", "GRAPH_DEF_VERSION", "GRAPH_DEF_VERSION_MIN_CONSUMER", diff --git a/tensorflow/python/platform/sysconfig.py b/tensorflow/python/platform/sysconfig.py index 1e3be40933..167dec6551 100644 --- a/tensorflow/python/platform/sysconfig.py +++ b/tensorflow/python/platform/sysconfig.py @@ -17,6 +17,8 @@ @@get_include @@get_lib +@@get_compile_flags +@@get_link_flags """ from __future__ import absolute_import from __future__ import division @@ -24,6 +26,7 @@ from __future__ import print_function import os.path as _os_path +from tensorflow.python.framework.versions import CXX11_ABI_FLAG as _CXX11_ABI_FLAG from tensorflow.python.util.all_util import remove_undocumented @@ -51,5 +54,31 @@ def get_lib(): import tensorflow as tf return _os_path.join(_os_path.dirname(tf.__file__)) + +def get_compile_flags(): + """Get the compilation flags for custom operators. + + Returns: + The compilation flags. + """ + flags = [] + flags.append('-I%s' % get_include()) + flags.append('-I%s/external/nsync/public' % get_include()) + if _CXX11_ABI_FLAG != -1: + flags.append('-D_GLIBCXX_USE_CXX11_ABI=%d' % _CXX11_ABI_FLAG) + return flags + + +def get_link_flags(): + """Get the link flags for custom operators. + + Returns: + The link flags. + """ + flags = [] + flags.append('-L%s' % get_lib()) + flags.append('-ltensorflow_framework') + return flags + _allowed_symbols = [] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/pywrap_tensorflow.py b/tensorflow/python/pywrap_tensorflow.py index 000ed8df8b..91373fa544 100644 --- a/tensorflow/python/pywrap_tensorflow.py +++ b/tensorflow/python/pywrap_tensorflow.py @@ -59,6 +59,7 @@ try: from tensorflow.python.pywrap_tensorflow_internal import __version__ from tensorflow.python.pywrap_tensorflow_internal import __git_version__ from tensorflow.python.pywrap_tensorflow_internal import __compiler_version__ + from tensorflow.python.pywrap_tensorflow_internal import __cxx11_abi_flag__ if _use_dlopen_global_flags: pywrap_dlopen_global_flags.reset_dlopen_flags() diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index f61f82e43e..bf7bc6a7c1 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -16,6 +16,10 @@ tf_module { name: "COMPILER_VERSION" mtype: "" } + member { + name: "CXX11_ABI_FLAG" + mtype: "" + } member { name: "ConditionalAccumulator" mtype: "" diff --git a/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt b/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt index 02dec04b9c..2f00aeac25 100644 --- a/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.sysconfig" tf_module { + member_method { + name: "get_compile_flags" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_include" argspec: "args=[], varargs=None, keywords=None, defaults=None" @@ -8,4 +12,8 @@ tf_module { name: "get_lib" argspec: "args=[], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_link_flags" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py index a7f8b5bb5f..616ec9fbe0 100755 --- a/tensorflow/tools/git/gen_git_source.py +++ b/tensorflow/tools/git/gen_git_source.py @@ -170,8 +170,16 @@ def write_version_info(filename, git_version): if b"\"" in git_version or b"\\" in git_version: git_version = "git_version_is_invalid" # do not cause build to fail! contents = """/* Generated by gen_git_source.py */ +#include const char* tf_git_version() {return "%s";} const char* tf_compiler_version() {return __VERSION__;} +const int tf_cxx11_abi_flag() { +#ifdef _GLIBCXX_USE_CXX11_ABI + return _GLIBCXX_USE_CXX11_ABI; +#else + return -1; +#endif +} """ % git_version open(filename, "w").write(contents) diff --git a/tensorflow/tools/git/gen_git_source.sh b/tensorflow/tools/git/gen_git_source.sh index 977fe16333..eb5e1abe15 100755 --- a/tensorflow/tools/git/gen_git_source.sh +++ b/tensorflow/tools/git/gen_git_source.sh @@ -26,7 +26,15 @@ if [[ $? != 0 ]]; then fi cat < ${OUTPUT_FILENAME} +#include const char* tf_git_version() {return "${GIT_VERSION}";} const char* tf_compiler_version() {return __VERSION__;} +const int tf_cxx11_abi_flag() { +#ifdef _GLIBCXX_USE_CXX11_ABI + return _GLIBCXX_USE_CXX11_ABI; +#else + return -1; +#endif +} EOF -- GitLab From 57b1c56214e88fc3b00f6ff518cb3277bfeb660a Mon Sep 17 00:00:00 2001 From: Alan Yee Date: Tue, 31 Oct 2017 15:17:46 -0700 Subject: [PATCH 570/573] Add deprecation notes (#12614) * Update lookup_ops.py Minor comment fix * Update metrics_ops.py Add deprecated notes * Update tensor_util.py Update deprecated note on remove_squeezable_dimensions * Update metric_ops.py Add deprecated notes --- .../framework/python/framework/tensor_util.py | 8 ++++---- .../contrib/metrics/python/ops/metric_ops.py | 20 ++++++++++++------- tensorflow/python/ops/lookup_ops.py | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/framework/python/framework/tensor_util.py b/tensorflow/contrib/framework/python/framework/tensor_util.py index 92a2a4ff2d..4e6eea8884 100644 --- a/tensorflow/contrib/framework/python/framework/tensor_util.py +++ b/tensorflow/contrib/framework/python/framework/tensor_util.py @@ -77,10 +77,10 @@ def reduce_sum_n(tensors, name=None): return tensors[0] return math_ops.add_n(tensors, name=name_scope) -@deprecated(None, - 'Please switch to tf.confusion_matrix.remove_squeezable_dimensions.' - 'Note that order of the inputs and outputs of labels and ' - 'predictions have also been switched.') +@deprecated( + None, "Please switch to remove_squeezable_dimensions from " + "tf.confusion_matrix. Note that the order of the inputs and outputs of " + "labels and predictions have also been switched.") def remove_squeezable_dimensions(predictions, labels, name=None): """Squeeze last dim if ranks of `predictions` and `labels` differ by 1. diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 177c4c53f7..dbfc0934ea 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -448,7 +448,8 @@ def streaming_mean_tensor(values, updates_collections=updates_collections, name=name) - +@deprecated(None, "Please switch to tf.metrics.accuracy. Note that the order " + "of the inputs of labels and predictions have been switched.") def streaming_accuracy(predictions, labels, weights=None, @@ -1122,7 +1123,8 @@ def streaming_curve_points(labels=None, return points, update_op - +@deprecated(None, "Please switch to tf.metrics.auc. Note that the order of " + "the inputs of labels and predictions have been switched.") def streaming_auc(predictions, labels, weights=None, @@ -1507,7 +1509,9 @@ def streaming_sensitivity_at_specificity(predictions, updates_collections=updates_collections, name=name) - +@deprecated( + None, "Please switch to tf.metrics.precision_at_thresholds. Note that the " + "order of of the inputs of labels and predictions have been switched.") def streaming_precision_at_thresholds(predictions, labels, thresholds, @@ -1566,7 +1570,9 @@ def streaming_precision_at_thresholds(predictions, updates_collections=updates_collections, name=name) - +@deprecated( + None, "Please switch to tf.metrics.recall_at_thresholds. Note that the " + "order of of the inputs of labels and predictions have been switched.") def streaming_recall_at_thresholds(predictions, labels, thresholds, @@ -1776,8 +1782,8 @@ def _at_k_name(name, k=None, class_id=None): return name -@deprecated('2016-11-08', 'Please use `streaming_sparse_recall_at_k`, ' - 'and reshape labels from [batch_size] to [batch_size, 1].') +@deprecated("2016-11-08", "Please use `streaming_sparse_recall_at_k`, " + "and reshape labels from [batch_size] to [batch_size, 1].") def streaming_recall_at_k(predictions, labels, k, @@ -2307,7 +2313,7 @@ def streaming_sparse_average_precision_at_top_k(top_k_predictions, updates_collections=updates_collections, name=name) - +@deprecated(None, "Please switch to tf.metrics.mean.") def streaming_mean_absolute_error(predictions, labels, weights=None, diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index 7f00344be2..fa58ffc37e 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -1011,7 +1011,7 @@ def index_table_from_tensor(vocabulary_list, Args: vocabulary_list: A 1-D `Tensor` that specifies the mapping of keys to - indices. Thetype of this object must be castable to `dtype`. + indices. The type of this object must be castable to `dtype`. num_oov_buckets: The number of out-of-vocabulary buckets. default_value: The value to use for out-of-vocabulary feature values. Defaults to -1. -- GitLab From 07a91dac5414298901c59be643e2d6eda324a557 Mon Sep 17 00:00:00 2001 From: nolan liu Date: Wed, 1 Nov 2017 14:35:18 +0800 Subject: [PATCH 571/573] make `gather` cpu kernel to be multiple threads. (#12246) * Change the gather op to multi-thread. * Modify the gather kernel of xla compiler in order to be compatible with multi-threads cpu kernel. * Add prefetch logic to gather op kernel. * Update the indention of gather op kernel code. * Update the gather kernel code for multiple thread. * Remove reference to ealier version of code in gather functor. * Change the framework_lite dep of gather_functor to framework. * Remove mutex guard in gather functor. --- tensorflow/core/kernels/BUILD | 2 +- tensorflow/core/kernels/gather_functor.cc | 2 +- tensorflow/core/kernels/gather_functor.h | 91 ++++++++++++------- .../core/kernels/gather_functor_gpu.cu.h | 3 +- tensorflow/core/kernels/gather_op.cc | 2 +- .../core/kernels/resource_variable_ops.cc | 2 +- 6 files changed, 63 insertions(+), 39 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 2aef1e3560..1cb7c97be4 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1098,7 +1098,7 @@ tf_kernel_library( visibility = [":friends"], deps = [ ":bounds_check", - "//tensorflow/core:framework_lite", + "//tensorflow/core:framework", "//third_party/eigen3", ], ) diff --git a/tensorflow/core/kernels/gather_functor.cc b/tensorflow/core/kernels/gather_functor.cc index 1b8be9b2ce..dde08b37ea 100644 --- a/tensorflow/core/kernels/gather_functor.cc +++ b/tensorflow/core/kernels/gather_functor.cc @@ -28,7 +28,7 @@ namespace functor { #define DECLARE_GPU_SPECS_INDEX(T, Index) \ template <> \ int64 GatherFunctor::operator()( \ - const GPUDevice& d, typename TTypes::ConstTensor Tparams, \ + OpKernelContext* ctx, typename TTypes::ConstTensor Tparams, \ typename TTypes::ConstFlat Tindices, \ typename TTypes::Tensor Tout); \ extern template struct GatherFunctor; diff --git a/tensorflow/core/kernels/gather_functor.h b/tensorflow/core/kernels/gather_functor.h index dfa1a5f1f9..1e429a037e 100644 --- a/tensorflow/core/kernels/gather_functor.h +++ b/tensorflow/core/kernels/gather_functor.h @@ -23,6 +23,8 @@ limitations under the License. #include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/platform/prefetch.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/util/work_sharder.h" namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; @@ -32,7 +34,8 @@ namespace functor { // Helper method to copy using memcpy. template -SliceIndex HandleCopies(typename TTypes::ConstTensor params, +SliceIndex HandleCopies(OpKernelContext* ctx, + typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, SliceIndex slice_elems, typename TTypes::Tensor out) { @@ -47,44 +50,64 @@ SliceIndex HandleCopies(typename TTypes::ConstTensor params, } // Compute slice_bytes here so that static knowledge is available const size_t slice_bytes = slice_elems * sizeof(T); - for (SliceIndex b = 0; b < batch_size; b++) { - for (SliceIndex i = 0; i < indices_size; i++) { - const SliceIndex i_next = i + 1; - const SliceIndex b_next = b + 1; - if (i_next < indices_size) { - port::prefetch(¶ms(b, indices(i_next), 0)); - port::prefetch(&out(b, i_next, 0)); - } else if (b_next < batch_size) { + auto worker_threads = ctx->device()->tensorflow_cpu_worker_threads(); + mutex mu; + // Store the value of invalidate index for printing error information, it's a shared variable. + SliceIndex result = -1; + auto work = [&] (int64 start, int64 end) { + SliceIndex batch_idx = static_cast(start / indices_size); + SliceIndex indices_idx = static_cast(start % indices_size); + SliceIndex batch_idx_end = static_cast(end / indices_size); + SliceIndex indices_idx_end = static_cast(end % indices_size); + + while ((batch_idx < batch_idx_end) || + (batch_idx == batch_idx_end && indices_idx < indices_idx_end)) { + SliceIndex i_next = indices_idx + 1; + SliceIndex b_next = batch_idx + 1; + if ((batch_idx == batch_idx_end && i_next < indices_idx_end) || + (i_next < indices_size)) { + port::prefetch(¶ms(batch_idx, indices(i_next), 0)); + port::prefetch(&out(batch_idx, i_next, 0)); + b_next = batch_idx; + } else if (b_next <= batch_idx_end) { port::prefetch(¶ms(b_next, indices(0), 0)); port::prefetch(&out(b_next, 0, 0)); + i_next = 0; + } + const Index index = internal::SubtleMustCopy(indices(indices_idx)); + if (!FastBoundsCheck(index, limit)) { + mutex_lock l(mu); + result = indices_idx; + return; } - // Grab the index and check its validity. An earlier version of the - // code checked it and then grabbed it from memory a second time, which - // was a security risk since it could have changed in between. - const Index index = internal::SubtleMustCopy(indices(i)); - if (!FastBoundsCheck(index, limit)) return i; // Copy using memcpy if possible, otherwise an Eigen loop // TODO(cwhipkey): avoid linking to framework to get Allocator (to improve // ahead-of-time compilation binary size). if (is_simple_type::value) { // Avoid auto-promotion to Index from SliceIndex by casting. - memcpy(out_base + (b * indices_size + i) * slice_elems, - params_base + (b * static_cast(limit) + + memcpy(out_base + (batch_idx * indices_size + indices_idx) * slice_elems, + params_base + (batch_idx * static_cast(limit) + static_cast(index)) * - slice_elems, + slice_elems, slice_bytes); } else { // For non-"simple" types (e.g. strings). - out.template chip<1>(i) = params.template chip<1>(index); + out.template chip<1>(indices_idx) = params.template chip<1>(index); } + indices_idx = i_next; + batch_idx = b_next; } - } - return -1; + }; + + Shard(worker_threads->num_threads, worker_threads->workers, batch_size*indices_size, + slice_elems * sizeof(T), work); + return result; } template struct GatherFunctorCPU { - int64 operator()(typename TTypes::ConstTensor params, + int64 operator()(OpKernelContext* ctx, + typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out) { const int64 N = indices.size(); @@ -94,16 +117,16 @@ struct GatherFunctorCPU { bool use_large = (slice_size > std::numeric_limits::max() || params.size() > std::numeric_limits::max() || N > std::numeric_limits::max()); -#define CALL(elems) \ - do { \ - if (use_large) { \ - bad_i = HandleCopies(params, indices, \ - slice_size, out); \ - } else { \ - const int32 small_slice = static_cast(slice_size); \ - bad_i = HandleCopies(params, indices, \ - small_slice, out); \ - } \ +#define CALL(elems) \ + do { \ + if (use_large) { \ + bad_i = HandleCopies(ctx, params, indices, \ + slice_size, out); \ + } else { \ + const int32 small_slice = static_cast(slice_size); \ + bad_i = HandleCopies(ctx, params, indices, \ + small_slice, out); \ + } \ } while (0) if (slice_size == 10) @@ -120,18 +143,18 @@ struct GatherFunctorCPU { template struct GatherFunctor { - int64 operator()(const Device& d, typename TTypes::ConstTensor params, + int64 operator()(OpKernelContext* ctx, typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out); }; template struct GatherFunctor { - int64 operator()(const CPUDevice& d, + int64 operator()(OpKernelContext* ctx, typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out) { - return GatherFunctorCPU()(params, indices, out); + return GatherFunctorCPU()(ctx, params, indices, out); } }; diff --git a/tensorflow/core/kernels/gather_functor_gpu.cu.h b/tensorflow/core/kernels/gather_functor_gpu.cu.h index e2384ef011..a50b51b54b 100644 --- a/tensorflow/core/kernels/gather_functor_gpu.cu.h +++ b/tensorflow/core/kernels/gather_functor_gpu.cu.h @@ -72,10 +72,11 @@ __global__ void GatherOpKernel(const T* params, const Index* indices, T* out, namespace functor { template struct GatherFunctor { - int64 operator()(const GPUDevice& d, + int64 operator()(OpKernelContext* ctx, typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out) { + const GPUDevice& d = ctx->eigen_gpu_device(); const int64 out_size = out.size(); if (out_size == 0) { // We need a check here since the CPU version does useful error checking diff --git a/tensorflow/core/kernels/gather_op.cc b/tensorflow/core/kernels/gather_op.cc index 7088005e73..239d5d2e99 100644 --- a/tensorflow/core/kernels/gather_op.cc +++ b/tensorflow/core/kernels/gather_op.cc @@ -106,7 +106,7 @@ class GatherOp : public OpKernel { auto out_flat = out->shaped({outer_size, N, inner_size}); functor::GatherFunctor functor; - int64 bad_i = functor(c->eigen_device(), params_flat, + int64 bad_i = functor(c, params_flat, indices_flat, out_flat); OP_REQUIRES( diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index 217fb3b781..0ae8a8fdbc 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -464,7 +464,7 @@ class ResourceGatherOp : public OpKernel { auto out_flat = out->shaped({1, N, out->NumElements() / N}); functor::GatherFunctor functor; - int64 bad_i = functor(c->eigen_device(), params_flat, + int64 bad_i = functor(c, params_flat, indices_flat, out_flat); OP_REQUIRES( -- GitLab From fa9d8aab41249cfc901338dfcb38cedb7ed1e603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20K=C3=B6ster?= Date: Tue, 31 Oct 2017 23:38:10 -0700 Subject: [PATCH 572/573] Add 'log_progress' argument for tf.estimator.Estimator's evaluate function (#13695) * Add argument for tf.estimator.Estimator's evaluate function * add log_progress argument to ._convert_eval_steps_to_hooks for TPU estimator * log only every 10th step if more than 100 iterations in _StopAfterNEvalsHook * ensure last step is logged and aim for 10 outputs total --- tensorflow/python/training/evaluation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/training/evaluation.py b/tensorflow/python/training/evaluation.py index fdcb9c2e90..b36444a14c 100644 --- a/tensorflow/python/training/evaluation.py +++ b/tensorflow/python/training/evaluation.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import time +import math from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -91,6 +92,9 @@ class _StopAfterNEvalsHook(session_run_hook.SessionRunHook): self._num_evals = num_evals self._evals_completed = None self._log_progress = log_progress + # Reduce logging frequency if there are 20 or more evaluations. + self._log_frequency = (1 if (num_evals is None or num_evals < 20) + else math.floor(num_evals / 10.)) def _set_evals_completed_tensor(self, updated_eval_step): self._evals_completed = updated_eval_step @@ -106,7 +110,9 @@ class _StopAfterNEvalsHook(session_run_hook.SessionRunHook): if self._num_evals is None: logging.info('Evaluation [%d]', evals_completed) else: - logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals) + if ((evals_completed % self._log_frequency) == 0 or + (self._num_evals == evals_completed)): + logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals) if self._num_evals is not None and evals_completed >= self._num_evals: run_context.request_stop() -- GitLab From 16b0bb095296fcfa17182aeae656a35faf70f36e Mon Sep 17 00:00:00 2001 From: loki der quaeler Date: Tue, 31 Oct 2017 23:40:37 -0700 Subject: [PATCH 573/573] Adding a feed for boolean tensors to TensorFlowInferenceInterface (#14059) * Sublime Text index-ignore file (a copy of .gitignore) * Adding the requested implementation to TensorFlowInferenceInterface * Removing Sublime Text .ignore file from remote repository * indeed there was --- .../android/TensorFlowInferenceInterface.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java index 80e03f2036..1f423a7a5b 100644 --- a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java +++ b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java @@ -282,6 +282,22 @@ public class TensorFlowInferenceInterface { // Methods for taking a native Tensor and filling it with values from Java arrays. + /** + * Given a source array with shape {@link dims} and content {@link src}, copy the contents into + * the input Tensor with name {@link inputName}. The source array {@link src} must have at least + * as many elements as that of the destination Tensor. If {@link src} has more elements than the + * destination has capacity, the copy is truncated. + */ + public void feed(String inputName, boolean[] src, long... dims) { + byte[] b = new byte[src.length]; + + for (int i = 0; i < src.length; i++) { + b[i] = src[i] ? (byte) 1 : (byte) 0; + } + + addFeed(inputName, Tensor.create(Boolean.class, dims, ByteBuffer.wrap(b))); + } + /** * Given a source array with shape {@link dims} and content {@link src}, copy the contents into * the input Tensor with name {@link inputName}. The source array {@link src} must have at least -- GitLab
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.4.0rc0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.368
tensorflow-1.4.0rc1CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.4.0rc1GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.368
tensorflow-1.2.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.2.0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.35.18
tensorflow-1.1.0CPU3.5MSVC 2015 update 3Cmake v3.6.3N/AN/A